diff --git a/.clang-tidy b/.clang-tidy index 860e7b3189f..532b0f37b81 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -22,6 +22,8 @@ Checks: '*, -bugprone-implicit-widening-of-multiplication-result, -bugprone-narrowing-conversions, -bugprone-not-null-terminated-result, + -bugprone-unchecked-optional-access, + -bugprone-assignment-in-if-condition, -cert-dcl16-c, -cert-err58-cpp, @@ -103,6 +105,7 @@ Checks: '*, -misc-no-recursion, -misc-non-private-member-variables-in-classes, + -misc-const-correctness, -modernize-avoid-c-arrays, -modernize-concat-nested-namespaces, @@ -114,6 +117,7 @@ Checks: '*, -modernize-use-nodiscard, -modernize-use-override, -modernize-use-trailing-return-type, + -modernize-macro-to-enum, -performance-inefficient-string-concatenation, -performance-no-int-to-ptr, @@ -135,6 +139,7 @@ Checks: '*, -readability-suspicious-call-argument, -readability-uppercase-literal-suffix, -readability-use-anyofallof, + -readability-simplify-boolean-expr, -zirkon-*, ' diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs new file mode 100644 index 00000000000..06e893fabb3 --- /dev/null +++ b/.git-blame-ignore-revs @@ -0,0 +1,15 @@ +# This is a file that can be used by git-blame to ignore some revisions. +# (git 2.23+, released in August 2019) +# +# Can be configured as follow: +# +# $ git config blame.ignoreRevsFile .git-blame-ignore-revs +# +# For more information you can look at git-blame(1) man page. + +# Changed tabs to spaces in code [#CLICKHOUSE-3] +137ad95929ee016cc6d3c03bccb5586941c163ff + +# dbms/ → src/ +# (though it is unlikely that you will see it in blame) +06446b4f08a142d6f1bc30664c47ded88ab51782 diff --git a/.github/workflows/backport_branches.yml b/.github/workflows/backport_branches.yml index 1c51d06f395..4c8d023f2ec 100644 --- a/.github/workflows/backport_branches.yml +++ b/.github/workflows/backport_branches.yml @@ -112,10 +112,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" ######################################################################################### #################################### ORDINARY BUILDS #################################### @@ -162,10 +160,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderDebAarch64: needs: [DockerHubPush] @@ -209,10 +205,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderDebAsan: needs: [DockerHubPush] @@ -254,10 +248,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderDebTsan: needs: [DockerHubPush] @@ -299,10 +291,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderDebDebug: needs: [DockerHubPush] @@ -344,10 +334,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderBinDarwin: needs: [DockerHubPush] @@ -391,10 +379,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderBinDarwinAarch64: needs: [DockerHubPush] @@ -438,10 +424,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" ############################################################################################ ##################################### Docker images ####################################### @@ -468,10 +452,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" ############################################################################################ ##################################### BUILD REPORTER ####################################### @@ -514,10 +496,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" BuilderSpecialReport: needs: @@ -554,10 +534,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" ############################################################################################## ########################### FUNCTIONAl STATELESS TESTS ####################################### @@ -594,10 +572,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" ############################################################################################## ############################ FUNCTIONAl STATEFUL TESTS ####################################### @@ -634,10 +610,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" ############################################################################################## ######################################### STRESS TESTS ####################################### @@ -677,10 +651,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" ############################################################################################# ############################# INTEGRATION TESTS ############################################# @@ -716,10 +688,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FinishCheck: needs: diff --git a/.github/workflows/cherry_pick.yml b/.github/workflows/cherry_pick.yml index e6a10479c7e..3e6f9e76c56 100644 --- a/.github/workflows/cherry_pick.yml +++ b/.github/workflows/cherry_pick.yml @@ -40,8 +40,6 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" diff --git a/.github/workflows/docs_check.yml b/.github/workflows/docs_check.yml index b50584a2c01..7a15e77becb 100644 --- a/.github/workflows/docs_check.yml +++ b/.github/workflows/docs_check.yml @@ -125,10 +125,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" DocsCheck: needs: DockerHubPush @@ -158,10 +156,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FinishCheck: needs: diff --git a/.github/workflows/docs_release.yml b/.github/workflows/docs_release.yml index e0fdb0c2f7b..da67edd4aa1 100644 --- a/.github/workflows/docs_release.yml +++ b/.github/workflows/docs_release.yml @@ -116,8 +116,6 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" diff --git a/.github/workflows/jepsen.yml b/.github/workflows/jepsen.yml index 1682cd1e812..a8b04af5773 100644 --- a/.github/workflows/jepsen.yml +++ b/.github/workflows/jepsen.yml @@ -36,8 +36,6 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index d3a303eb7ab..6f2fd5d678d 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -112,10 +112,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" CompatibilityCheck: needs: [BuilderDebRelease] @@ -146,10 +144,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" SharedBuildSmokeTest: needs: [BuilderDebShared] @@ -180,10 +176,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" ######################################################################################### #################################### ORDINARY BUILDS #################################### @@ -230,10 +224,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" BuilderDebAarch64: needs: [DockerHubPush] @@ -273,10 +265,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderBinRelease: needs: [DockerHubPush] @@ -320,56 +310,9 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - # BuilderBinGCC: - # needs: [DockerHubPush] - # runs-on: [self-hosted, builder] - # steps: - # - name: Set envs - # run: | - # cat >> "$GITHUB_ENV" << 'EOF' - # TEMP_PATH=${{runner.temp}}/build_check - # IMAGES_PATH=${{runner.temp}}/images_path - # REPO_COPY=${{runner.temp}}/build_check/ClickHouse - # CACHES_PATH=${{runner.temp}}/../ccaches - # BUILD_NAME=binary_gcc - # EOF - # - name: Download changed images - # uses: actions/download-artifact@v2 - # with: - # name: changed_images - # path: ${{ env.IMAGES_PATH }} - # - name: Clear repository - # run: | - # sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - # - name: Check out repository code - # uses: actions/checkout@v2 - # - name: Build - # run: | - # git -C "$GITHUB_WORKSPACE" submodule sync --recursive - # git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 - # sudo rm -fr "$TEMP_PATH" - # mkdir -p "$TEMP_PATH" - # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - # - name: Upload build URLs to artifacts - # if: ${{ success() || failure() }} - # uses: actions/upload-artifact@v2 - # with: - # name: ${{ env.BUILD_URLS }} - # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - # - name: Cleanup - # if: always() - # run: | - # # shellcheck disable=SC2046 - # docker kill $(docker ps -q) ||: - # # shellcheck disable=SC2046 - # docker rm -f $(docker ps -a -q) ||: - # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderDebAsan: needs: [DockerHubPush] runs-on: [self-hosted, builder] @@ -410,10 +353,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderDebUBsan: needs: [DockerHubPush] @@ -455,10 +396,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderDebTsan: needs: [DockerHubPush] @@ -500,10 +439,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderDebMsan: needs: [DockerHubPush] @@ -545,10 +482,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderDebDebug: needs: [DockerHubPush] @@ -590,10 +525,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" ########################################################################################## ##################################### SPECIAL BUILDS ##################################### @@ -638,10 +571,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderBinClangTidy: needs: [DockerHubPush] @@ -683,10 +614,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderBinDarwin: needs: [DockerHubPush] @@ -730,10 +659,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderBinAarch64: needs: [DockerHubPush] @@ -777,10 +704,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderBinFreeBSD: needs: [DockerHubPush] @@ -824,10 +749,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderBinDarwinAarch64: needs: [DockerHubPush] @@ -871,10 +794,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderBinPPC64: needs: [DockerHubPush] @@ -918,10 +839,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderBinAmd64SSE2: needs: [DockerHubPush] @@ -965,10 +884,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" ############################################################################################ ##################################### Docker images ####################################### @@ -995,10 +912,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" ############################################################################################ ##################################### BUILD REPORTER ####################################### @@ -1045,10 +960,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" BuilderSpecialReport: needs: @@ -1092,10 +1005,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" ############################################################################################## ########################### FUNCTIONAl STATELESS TESTS ####################################### @@ -1132,10 +1043,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestReleaseDatabaseOrdinary: needs: [BuilderDebRelease] @@ -1169,10 +1078,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestReleaseDatabaseReplicated0: needs: [BuilderDebRelease] @@ -1208,10 +1115,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestReleaseDatabaseReplicated1: needs: [BuilderDebRelease] @@ -1247,10 +1152,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestReleaseS3: needs: [BuilderDebRelease] @@ -1284,10 +1187,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestAarch64: needs: [BuilderDebAarch64] @@ -1321,10 +1222,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestAsan0: needs: [BuilderDebAsan] @@ -1360,10 +1259,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestAsan1: needs: [BuilderDebAsan] @@ -1399,10 +1296,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestTsan0: needs: [BuilderDebTsan] @@ -1438,10 +1333,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestTsan1: needs: [BuilderDebTsan] @@ -1477,10 +1370,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestTsan2: needs: [BuilderDebTsan] @@ -1516,10 +1407,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestUBsan: needs: [BuilderDebUBsan] @@ -1553,10 +1442,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestMsan0: needs: [BuilderDebMsan] @@ -1592,10 +1479,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestMsan1: needs: [BuilderDebMsan] @@ -1631,10 +1516,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestMsan2: needs: [BuilderDebMsan] @@ -1670,10 +1553,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestDebug0: needs: [BuilderDebDebug] @@ -1709,10 +1590,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestDebug1: needs: [BuilderDebDebug] @@ -1748,10 +1627,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestDebug2: needs: [BuilderDebDebug] @@ -1787,10 +1664,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" ############################################################################################## ############################ FUNCTIONAl STATEFUL TESTS ####################################### @@ -1827,10 +1702,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatefulTestAarch64: needs: [BuilderDebAarch64] @@ -1864,10 +1737,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatefulTestAsan: needs: [BuilderDebAsan] @@ -1901,10 +1772,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatefulTestTsan: needs: [BuilderDebTsan] @@ -1938,10 +1807,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatefulTestMsan: needs: [BuilderDebMsan] @@ -1975,10 +1842,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatefulTestUBsan: needs: [BuilderDebUBsan] @@ -2012,10 +1877,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatefulTestDebug: needs: [BuilderDebDebug] @@ -2049,10 +1912,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" ############################################################################################## ######################################### STRESS TESTS ####################################### @@ -2088,10 +1949,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" StressTestTsan: needs: [BuilderDebTsan] @@ -2128,10 +1987,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" StressTestMsan: needs: [BuilderDebMsan] @@ -2164,10 +2021,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" StressTestUBsan: needs: [BuilderDebUBsan] @@ -2200,10 +2055,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" StressTestDebug: needs: [BuilderDebDebug] @@ -2236,10 +2089,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" ############################################################################################# ############################# INTEGRATION TESTS ############################################# @@ -2277,10 +2128,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" IntegrationTestsAsan1: needs: [BuilderDebAsan] @@ -2315,10 +2164,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" IntegrationTestsAsan2: needs: [BuilderDebAsan] @@ -2353,10 +2200,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" IntegrationTestsTsan0: needs: [BuilderDebTsan] @@ -2391,10 +2236,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" IntegrationTestsTsan1: needs: [BuilderDebTsan] @@ -2429,10 +2272,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" IntegrationTestsTsan2: needs: [BuilderDebTsan] @@ -2467,10 +2308,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" IntegrationTestsTsan3: needs: [BuilderDebTsan] @@ -2505,10 +2344,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" IntegrationTestsRelease0: needs: [BuilderDebRelease] @@ -2543,10 +2380,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" IntegrationTestsRelease1: needs: [BuilderDebRelease] @@ -2581,10 +2416,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" ############################################################################################## ##################################### AST FUZZERS ############################################ @@ -2620,10 +2453,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" ASTFuzzerTestTsan: needs: [BuilderDebTsan] @@ -2656,10 +2487,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" ASTFuzzerTestUBSan: needs: [BuilderDebUBsan] @@ -2692,10 +2521,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" ASTFuzzerTestMSan: needs: [BuilderDebMsan] @@ -2728,10 +2555,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" ASTFuzzerTestDebug: needs: [BuilderDebDebug] @@ -2764,10 +2589,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" ############################################################################################# #################################### UNIT TESTS ############################################# @@ -2803,10 +2626,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" UnitTestsReleaseClang: needs: [BuilderBinRelease] @@ -2839,10 +2660,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" # UnitTestsReleaseGCC: # needs: [BuilderBinGCC] @@ -2875,10 +2694,8 @@ jobs: # - name: Cleanup # if: always() # run: | - # # shellcheck disable=SC2046 - # docker kill $(docker ps -q) ||: - # # shellcheck disable=SC2046 - # docker rm -f $(docker ps -a -q) ||: + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" UnitTestsTsan: needs: [BuilderDebTsan] @@ -2911,10 +2728,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" UnitTestsMsan: needs: [BuilderDebMsan] @@ -2947,10 +2762,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" UnitTestsUBsan: needs: [BuilderDebUBsan] @@ -2983,10 +2796,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" ############################################################################################# #################################### PERFORMANCE TESTS ###################################### @@ -3024,10 +2835,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" PerformanceComparisonX86-1: needs: [BuilderDebRelease] @@ -3062,10 +2871,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" PerformanceComparisonX86-2: needs: [BuilderDebRelease] @@ -3100,10 +2907,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" PerformanceComparisonX86-3: needs: [BuilderDebRelease] @@ -3138,10 +2943,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FinishCheck: needs: diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index e712ada1551..801f7eda94a 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -119,8 +119,6 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 3f4e5d7bb00..24a1c6bb714 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -141,10 +141,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FastTest: needs: DockerHubPush @@ -177,10 +175,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" CompatibilityCheck: needs: [BuilderDebRelease] @@ -211,10 +207,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" SharedBuildSmokeTest: needs: [BuilderDebShared] @@ -245,10 +239,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" ######################################################################################### #################################### ORDINARY BUILDS #################################### @@ -295,10 +287,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" BuilderBinRelease: needs: [DockerHubPush, FastTest, StyleCheck] @@ -340,10 +330,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderDebAarch64: needs: [DockerHubPush, FastTest, StyleCheck] @@ -387,10 +375,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderDebAsan: needs: [DockerHubPush, FastTest, StyleCheck] @@ -432,10 +418,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderDebUBsan: needs: [DockerHubPush, FastTest, StyleCheck] @@ -477,10 +461,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderDebTsan: needs: [DockerHubPush, FastTest, StyleCheck] @@ -522,10 +504,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderDebMsan: needs: [DockerHubPush, FastTest, StyleCheck] @@ -567,10 +547,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderDebDebug: needs: [DockerHubPush, FastTest, StyleCheck] @@ -612,10 +590,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" ########################################################################################## ##################################### SPECIAL BUILDS ##################################### @@ -660,10 +636,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderBinClangTidy: needs: [DockerHubPush, FastTest, StyleCheck] @@ -705,10 +679,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderBinDarwin: needs: [DockerHubPush, FastTest, StyleCheck] @@ -750,10 +722,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderBinAarch64: needs: [DockerHubPush, FastTest, StyleCheck] @@ -795,10 +765,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderBinFreeBSD: needs: [DockerHubPush, FastTest, StyleCheck] @@ -840,10 +808,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderBinDarwinAarch64: needs: [DockerHubPush, FastTest, StyleCheck] @@ -885,10 +851,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderBinPPC64: needs: [DockerHubPush, FastTest, StyleCheck] @@ -930,10 +894,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderBinAmd64SSE2: needs: [DockerHubPush, FastTest, StyleCheck] @@ -975,10 +937,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" ############################################################################################ ##################################### Docker images ####################################### @@ -1005,10 +965,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" ############################################################################################ ##################################### BUILD REPORTER ####################################### @@ -1055,10 +1013,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" BuilderSpecialReport: needs: @@ -1103,10 +1059,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" ############################################################################################## ########################### FUNCTIONAl STATELESS TESTS ####################################### @@ -1143,10 +1097,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestReleaseDatabaseReplicated0: needs: [BuilderDebRelease] @@ -1182,10 +1134,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestReleaseDatabaseReplicated1: needs: [BuilderDebRelease] @@ -1221,10 +1171,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestReleaseWideParts: needs: [BuilderDebRelease] @@ -1258,10 +1206,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestReleaseS3: needs: [BuilderDebRelease] @@ -1295,10 +1241,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestS3Debug0: needs: [BuilderDebDebug] @@ -1334,8 +1278,8 @@ jobs: - name: Cleanup if: always() run: | - docker kill "$(docker ps -q)" ||: - docker rm -f "$(docker ps -a -q)" ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestS3Debug1: needs: [BuilderDebDebug] @@ -1371,8 +1315,8 @@ jobs: - name: Cleanup if: always() run: | - docker kill "$(docker ps -q)" ||: - docker rm -f "$(docker ps -a -q)" ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestS3Debug2: needs: [BuilderDebDebug] @@ -1408,8 +1352,8 @@ jobs: - name: Cleanup if: always() run: | - docker kill "$(docker ps -q)" ||: - docker rm -f "$(docker ps -a -q)" ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestS3Tsan0: needs: [BuilderDebTsan] @@ -1445,8 +1389,8 @@ jobs: - name: Cleanup if: always() run: | - docker kill "$(docker ps -q)" ||: - docker rm -f "$(docker ps -a -q)" ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestS3Tsan1: needs: [BuilderDebTsan] @@ -1482,8 +1426,8 @@ jobs: - name: Cleanup if: always() run: | - docker kill "$(docker ps -q)" ||: - docker rm -f "$(docker ps -a -q)" ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestS3Tsan2: needs: [BuilderDebTsan] @@ -1519,8 +1463,8 @@ jobs: - name: Cleanup if: always() run: | - docker kill "$(docker ps -q)" ||: - docker rm -f "$(docker ps -a -q)" ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestAarch64: needs: [BuilderDebAarch64] @@ -1554,10 +1498,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestAsan0: needs: [BuilderDebAsan] @@ -1593,10 +1535,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestAsan1: needs: [BuilderDebAsan] @@ -1632,10 +1572,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestTsan0: needs: [BuilderDebTsan] @@ -1671,10 +1609,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestTsan1: needs: [BuilderDebTsan] @@ -1710,10 +1646,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestTsan2: needs: [BuilderDebTsan] @@ -1749,10 +1683,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestUBsan: needs: [BuilderDebUBsan] @@ -1786,10 +1718,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestMsan0: needs: [BuilderDebMsan] @@ -1825,10 +1755,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestMsan1: needs: [BuilderDebMsan] @@ -1864,10 +1792,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestMsan2: needs: [BuilderDebMsan] @@ -1903,10 +1829,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestDebug0: needs: [BuilderDebDebug] @@ -1942,10 +1866,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestDebug1: needs: [BuilderDebDebug] @@ -1981,10 +1903,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestDebug2: needs: [BuilderDebDebug] @@ -2020,10 +1940,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestFlakyCheck: needs: [BuilderDebAsan] @@ -2057,10 +1975,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" TestsBugfixCheck: runs-on: [self-hosted, stress-tester] @@ -2104,10 +2020,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" ############################################################################################## ############################ FUNCTIONAl STATEFUL TESTS ####################################### @@ -2144,10 +2058,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatefulTestAarch64: needs: [BuilderDebAarch64] @@ -2181,10 +2093,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatefulTestAsan: needs: [BuilderDebAsan] @@ -2218,10 +2128,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatefulTestTsan: needs: [BuilderDebTsan] @@ -2255,10 +2163,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatefulTestMsan: needs: [BuilderDebMsan] @@ -2292,10 +2198,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatefulTestUBsan: needs: [BuilderDebUBsan] @@ -2329,10 +2233,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatefulTestDebug: needs: [BuilderDebDebug] @@ -2366,10 +2268,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" ############################################################################################## ######################################### STRESS TESTS ####################################### @@ -2405,10 +2305,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" StressTestTsan: needs: [BuilderDebTsan] @@ -2445,10 +2343,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" StressTestMsan: needs: [BuilderDebMsan] @@ -2481,10 +2377,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" StressTestUBsan: needs: [BuilderDebUBsan] @@ -2517,10 +2411,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" StressTestDebug: needs: [BuilderDebDebug] @@ -2553,10 +2445,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" ############################################################################################## ##################################### AST FUZZERS ############################################ @@ -2592,10 +2482,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" ASTFuzzerTestTsan: needs: [BuilderDebTsan] @@ -2628,10 +2516,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" ASTFuzzerTestUBSan: needs: [BuilderDebUBsan] @@ -2664,10 +2550,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" ASTFuzzerTestMSan: needs: [BuilderDebMsan] @@ -2700,10 +2584,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" ASTFuzzerTestDebug: needs: [BuilderDebDebug] @@ -2736,10 +2618,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" ############################################################################################# ############################# INTEGRATION TESTS ############################################# @@ -2777,10 +2657,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" IntegrationTestsAsan1: needs: [BuilderDebAsan] @@ -2815,10 +2693,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" IntegrationTestsAsan2: needs: [BuilderDebAsan] @@ -2853,10 +2729,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" IntegrationTestsTsan0: needs: [BuilderDebTsan] @@ -2891,10 +2765,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" IntegrationTestsTsan1: needs: [BuilderDebTsan] @@ -2929,10 +2801,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" IntegrationTestsTsan2: needs: [BuilderDebTsan] @@ -2967,10 +2837,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" IntegrationTestsTsan3: needs: [BuilderDebTsan] @@ -3005,10 +2873,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" IntegrationTestsRelease0: needs: [BuilderDebRelease] @@ -3043,10 +2909,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" IntegrationTestsRelease1: needs: [BuilderDebRelease] @@ -3081,10 +2945,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" IntegrationTestsFlakyCheck: needs: [BuilderDebAsan] @@ -3117,10 +2979,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" ############################################################################################# #################################### UNIT TESTS ############################################# @@ -3156,10 +3016,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" UnitTestsReleaseClang: needs: [BuilderBinRelease] @@ -3192,10 +3050,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" UnitTestsTsan: needs: [BuilderDebTsan] @@ -3228,10 +3084,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" UnitTestsMsan: needs: [BuilderDebMsan] @@ -3264,10 +3118,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" UnitTestsUBsan: needs: [BuilderDebUBsan] @@ -3300,10 +3152,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" ############################################################################################# #################################### PERFORMANCE TESTS ###################################### @@ -3341,10 +3191,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" PerformanceComparisonX86-1: needs: [BuilderDebRelease] @@ -3379,10 +3227,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" PerformanceComparisonX86-2: needs: [BuilderDebRelease] @@ -3417,10 +3263,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" PerformanceComparisonX86-3: needs: [BuilderDebRelease] @@ -3455,10 +3299,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" PerformanceComparisonAarch-0: needs: [BuilderDebAarch64] @@ -3493,10 +3335,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" PerformanceComparisonAarch-1: needs: [BuilderDebAarch64] @@ -3531,10 +3371,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" PerformanceComparisonAarch-2: needs: [BuilderDebAarch64] @@ -3569,10 +3407,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" PerformanceComparisonAarch-3: needs: [BuilderDebAarch64] @@ -3607,10 +3443,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" ############################################################################################# ###################################### JEPSEN TESTS ######################################### diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ae905aa62ba..0b0f125d641 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -30,10 +30,11 @@ jobs: cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" cd "$REPO_COPY" # Download and push packages to artifactory - python3 ./tests/ci/push_to_artifactory.py --release "${{ github.ref }}" \ - --commit '${{ github.sha }}' --artifactory-url "${{ secrets.JFROG_ARTIFACTORY_URL }}" --all + python3 ./tests/ci/push_to_artifactory.py --release '${{ github.ref }}' \ + --commit '${{ github.sha }}' --artifactory-url '${{ secrets.JFROG_ARTIFACTORY_URL }}' --all # Download macos binaries to ${{runner.temp}}/download_binary - python3 ./tests/ci/download_binary.py binary_darwin binary_darwin_aarch64 + python3 ./tests/ci/download_binary.py --version '${{ github.ref }}' \ + --commit '${{ github.sha }}' binary_darwin binary_darwin_aarch64 mv '${{runner.temp}}/download_binary/'clickhouse-* '${{runner.temp}}/push_to_artifactory' - name: Upload packages to release assets uses: svenstaro/upload-release-action@v2 @@ -65,8 +66,6 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index f579d1fee63..8f42ca92646 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -103,10 +103,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" ######################################################################################### #################################### ORDINARY BUILDS #################################### @@ -153,10 +151,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderDebAarch64: needs: [DockerHubPush] @@ -196,10 +192,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderDebAsan: needs: [DockerHubPush] @@ -241,10 +235,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderDebUBsan: needs: [DockerHubPush] @@ -286,10 +278,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderDebTsan: needs: [DockerHubPush] @@ -331,10 +321,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderDebMsan: needs: [DockerHubPush] @@ -376,10 +364,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderDebDebug: needs: [DockerHubPush] @@ -421,10 +407,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderBinDarwin: needs: [DockerHubPush] @@ -468,10 +452,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderBinDarwinAarch64: needs: [DockerHubPush] @@ -515,10 +497,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" ############################################################################################ ##################################### Docker images ####################################### @@ -545,10 +525,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" ############################################################################################ ##################################### BUILD REPORTER ####################################### @@ -594,10 +572,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" BuilderSpecialReport: needs: @@ -634,10 +610,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" ############################################################################################## ########################### FUNCTIONAl STATELESS TESTS ####################################### @@ -674,10 +648,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestAarch64: needs: [BuilderDebAarch64] @@ -711,10 +683,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestAsan0: needs: [BuilderDebAsan] @@ -750,10 +720,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestAsan1: needs: [BuilderDebAsan] @@ -789,10 +757,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestTsan0: needs: [BuilderDebTsan] @@ -828,10 +794,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestTsan1: needs: [BuilderDebTsan] @@ -867,10 +831,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestTsan2: needs: [BuilderDebTsan] @@ -906,10 +868,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestUBsan: needs: [BuilderDebUBsan] @@ -943,10 +903,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestMsan0: needs: [BuilderDebMsan] @@ -982,10 +940,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestMsan1: needs: [BuilderDebMsan] @@ -1021,10 +977,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestMsan2: needs: [BuilderDebMsan] @@ -1060,10 +1014,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestDebug0: needs: [BuilderDebDebug] @@ -1099,10 +1051,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestDebug1: needs: [BuilderDebDebug] @@ -1138,10 +1088,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestDebug2: needs: [BuilderDebDebug] @@ -1177,10 +1125,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" ############################################################################################## ############################ FUNCTIONAl STATEFUL TESTS ####################################### @@ -1217,10 +1163,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatefulTestAarch64: needs: [BuilderDebAarch64] @@ -1254,10 +1198,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatefulTestAsan: needs: [BuilderDebAsan] @@ -1291,10 +1233,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatefulTestTsan: needs: [BuilderDebTsan] @@ -1328,10 +1268,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatefulTestMsan: needs: [BuilderDebMsan] @@ -1365,10 +1303,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatefulTestUBsan: needs: [BuilderDebUBsan] @@ -1402,10 +1338,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatefulTestDebug: needs: [BuilderDebDebug] @@ -1439,10 +1373,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" ############################################################################################## ######################################### STRESS TESTS ####################################### @@ -1478,10 +1410,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" StressTestTsan: needs: [BuilderDebTsan] @@ -1518,10 +1448,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" StressTestMsan: needs: [BuilderDebMsan] @@ -1554,10 +1482,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" StressTestUBsan: needs: [BuilderDebUBsan] @@ -1590,10 +1516,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" StressTestDebug: needs: [BuilderDebDebug] @@ -1626,10 +1550,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" ############################################################################################# ############################# INTEGRATION TESTS ############################################# @@ -1667,10 +1589,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" IntegrationTestsAsan1: needs: [BuilderDebAsan] @@ -1705,10 +1625,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" IntegrationTestsAsan2: needs: [BuilderDebAsan] @@ -1743,10 +1661,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" IntegrationTestsTsan0: needs: [BuilderDebTsan] @@ -1781,10 +1697,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" IntegrationTestsTsan1: needs: [BuilderDebTsan] @@ -1819,10 +1733,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" IntegrationTestsTsan2: needs: [BuilderDebTsan] @@ -1857,10 +1769,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" IntegrationTestsTsan3: needs: [BuilderDebTsan] @@ -1895,10 +1805,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" IntegrationTestsRelease0: needs: [BuilderDebRelease] @@ -1933,10 +1841,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" IntegrationTestsRelease1: needs: [BuilderDebRelease] @@ -1971,10 +1877,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FinishCheck: needs: diff --git a/.github/workflows/tags_stable.yml b/.github/workflows/tags_stable.yml index 9711f7688cb..a9172a8a2e2 100644 --- a/.github/workflows/tags_stable.yml +++ b/.github/workflows/tags_stable.yml @@ -43,6 +43,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.ROBOT_CLICKHOUSE_COMMIT_TOKEN }} run: | ./utils/list-versions/list-versions.sh > ./utils/list-versions/version_date.tsv + ./utils/list-versions/update-docker-version.sh GID=$(id -g "${UID}") docker run -u "${UID}:${GID}" -e PYTHONUNBUFFERED=1 \ --volume="${GITHUB_WORKSPACE}:/ClickHouse" clickhouse/style-test \ diff --git a/.github/workflows/woboq.yml b/.github/workflows/woboq.yml index dd29131e9dc..b928a4a8d3d 100644 --- a/.github/workflows/woboq.yml +++ b/.github/workflows/woboq.yml @@ -37,8 +37,6 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" diff --git a/.gitignore b/.gitignore index e517dfd63c2..dd632eba85d 100644 --- a/.gitignore +++ b/.gitignore @@ -58,6 +58,10 @@ cmake_install.cmake CTestTestfile.cmake *.a *.o +*.so +*.dll +*.lib +*.dylib cmake-build-* # Python cache diff --git a/base/base/ReplxxLineReader.cpp b/base/base/ReplxxLineReader.cpp index b7c18110503..75c48f690f8 100644 --- a/base/base/ReplxxLineReader.cpp +++ b/base/base/ReplxxLineReader.cpp @@ -220,6 +220,35 @@ ReplxxLineReader::ReplxxLineReader( rx.bind_key(Replxx::KEY::control('W'), [this](char32_t code) { return rx.invoke(Replxx::ACTION::KILL_TO_WHITESPACE_ON_LEFT, code); }); rx.bind_key(Replxx::KEY::meta('E'), [this](char32_t) { openEditor(); return Replxx::ACTION_RESULT::CONTINUE; }); + + /// readline insert-comment + auto insert_comment_action = [this](char32_t code) + { + replxx::Replxx::State state(rx.get_state()); + const char * line = state.text(); + const char * line_end = line + strlen(line); + + std::string commented_line; + if (std::find(line, line_end, '\n') != line_end) + { + /// If query has multiple lines, multiline comment is used over + /// commenting each line separately for easier uncomment (though + /// with invoking editor it is simpler to uncomment multiple lines) + /// + /// Note, that using multiline comment is OK even with nested + /// comments, since nested comments are supported. + commented_line = fmt::format("/* {} */", state.text()); + } + else + { + // In a simplest case use simple comment. + commented_line = fmt::format("-- {}", state.text()); + } + rx.set_state(replxx::Replxx::State(commented_line.c_str(), commented_line.size())); + + return rx.invoke(Replxx::ACTION::COMMIT_LINE, code); + }; + rx.bind_key(Replxx::KEY::meta('#'), insert_comment_action); } ReplxxLineReader::~ReplxxLineReader() diff --git a/base/base/bit_cast.h b/base/base/bit_cast.h index d1246b45590..b2b6915764d 100644 --- a/base/base/bit_cast.h +++ b/base/base/bit_cast.h @@ -5,8 +5,9 @@ #include -/** \brief Returns value `from` converted to type `To` while retaining bit representation. - * `To` and `From` must satisfy `CopyConstructible`. +/** Returns value `from` converted to type `To` while retaining bit representation. + * `To` and `From` must satisfy `CopyConstructible`. + * In contrast to std::bit_cast can cast types of different width. */ template std::decay_t bit_cast(const From & from) @@ -15,13 +16,3 @@ std::decay_t bit_cast(const From & from) memcpy(static_cast(&res), &from, std::min(sizeof(res), sizeof(from))); return res; } - -/** \brief Returns value `from` converted to type `To` while retaining bit representation. - * `To` and `From` must satisfy `CopyConstructible`. - */ -template -std::decay_t safe_bit_cast(const From & from) -{ - static_assert(sizeof(To) == sizeof(From), "bit cast on types of different width"); - return bit_cast(from); -} diff --git a/base/base/phdr_cache.h b/base/base/phdr_cache.h index b522710c4c4..a6e129db34d 100644 --- a/base/base/phdr_cache.h +++ b/base/base/phdr_cache.h @@ -8,6 +8,7 @@ * As a drawback, this only works if no dynamic object unloading happens after this point. * This function is thread-safe. You should call it to update cache after loading new shared libraries. * Otherwise exception handling from dlopened libraries won't work (will call std::terminate immediately). + * NOTE: dlopen is forbidden in our code. * * NOTE: It is disabled with Thread Sanitizer because TSan can only use original "dl_iterate_phdr" function. */ diff --git a/cmake/clang_tidy.cmake b/cmake/clang_tidy.cmake index fc25c68b11a..200282234ca 100644 --- a/cmake/clang_tidy.cmake +++ b/cmake/clang_tidy.cmake @@ -3,7 +3,7 @@ option (ENABLE_CLANG_TIDY "Use clang-tidy static analyzer" OFF) if (ENABLE_CLANG_TIDY) - find_program (CLANG_TIDY_PATH NAMES "clang-tidy" "clang-tidy-14" "clang-tidy-13" "clang-tidy-12") + find_program (CLANG_TIDY_PATH NAMES "clang-tidy" "clang-tidy-15" "clang-tidy-14" "clang-tidy-13" "clang-tidy-12") if (CLANG_TIDY_PATH) message(STATUS diff --git a/cmake/target.cmake b/cmake/target.cmake index 0fb5e8a20de..ae360758701 100644 --- a/cmake/target.cmake +++ b/cmake/target.cmake @@ -45,6 +45,7 @@ if (CMAKE_CROSSCOMPILING) endif () if (USE_MUSL) + # use of undeclared identifier 'PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP' set (ENABLE_SENTRY OFF CACHE INTERNAL "") set (ENABLE_ODBC OFF CACHE INTERNAL "") set (ENABLE_GRPC OFF CACHE INTERNAL "") diff --git a/contrib/c-ares-cmake/CMakeLists.txt b/contrib/c-ares-cmake/CMakeLists.txt index 603c1f8b65c..4b1170f9dd1 100644 --- a/contrib/c-ares-cmake/CMakeLists.txt +++ b/contrib/c-ares-cmake/CMakeLists.txt @@ -1,35 +1,95 @@ -# Choose to build static or shared library for c-ares. -if (USE_STATIC_LIBRARIES) - set(CARES_STATIC ON CACHE BOOL "" FORCE) - set(CARES_SHARED OFF CACHE BOOL "" FORCE) -else () - set(CARES_STATIC OFF CACHE BOOL "" FORCE) - set(CARES_SHARED ON CACHE BOOL "" FORCE) -endif () +set(LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/c-ares") -# Disable looking for libnsl on a platforms that has gethostbyname in glibc -# -# c-ares searching for gethostbyname in the libnsl library, however in the -# version that shipped with gRPC it doing it wrong [1], since it uses -# CHECK_LIBRARY_EXISTS(), which will return TRUE even if the function exists in -# another dependent library. The upstream already contains correct macro [2], -# but it is not included in gRPC (even upstream gRPC, not the one that is -# shipped with clickhousee). -# -# [1]: https://github.com/c-ares/c-ares/blob/e982924acee7f7313b4baa4ee5ec000c5e373c30/CMakeLists.txt#L125 -# [2]: https://github.com/c-ares/c-ares/blob/44fbc813685a1fa8aa3f27fcd7544faf612d376a/CMakeLists.txt#L146 -# -# And because if you by some reason have libnsl [3] installed, clickhouse will -# reject to start w/o it. While this is completelly different library. -# -# [3]: https://packages.debian.org/bullseye/libnsl2 -if (NOT CMAKE_SYSTEM_NAME STREQUAL "SunOS") - set(HAVE_LIBNSL OFF CACHE BOOL "" FORCE) +# Generated from contrib/c-ares/src/lib/Makefile.inc +SET(SRCS + "${LIBRARY_DIR}/src/lib/ares__addrinfo2hostent.c" + "${LIBRARY_DIR}/src/lib/ares__addrinfo_localhost.c" + "${LIBRARY_DIR}/src/lib/ares__close_sockets.c" + "${LIBRARY_DIR}/src/lib/ares__get_hostent.c" + "${LIBRARY_DIR}/src/lib/ares__parse_into_addrinfo.c" + "${LIBRARY_DIR}/src/lib/ares__readaddrinfo.c" + "${LIBRARY_DIR}/src/lib/ares__sortaddrinfo.c" + "${LIBRARY_DIR}/src/lib/ares__read_line.c" + "${LIBRARY_DIR}/src/lib/ares__timeval.c" + "${LIBRARY_DIR}/src/lib/ares_android.c" + "${LIBRARY_DIR}/src/lib/ares_cancel.c" + "${LIBRARY_DIR}/src/lib/ares_data.c" + "${LIBRARY_DIR}/src/lib/ares_destroy.c" + "${LIBRARY_DIR}/src/lib/ares_expand_name.c" + "${LIBRARY_DIR}/src/lib/ares_expand_string.c" + "${LIBRARY_DIR}/src/lib/ares_fds.c" + "${LIBRARY_DIR}/src/lib/ares_free_hostent.c" + "${LIBRARY_DIR}/src/lib/ares_free_string.c" + "${LIBRARY_DIR}/src/lib/ares_freeaddrinfo.c" + "${LIBRARY_DIR}/src/lib/ares_getaddrinfo.c" + "${LIBRARY_DIR}/src/lib/ares_getenv.c" + "${LIBRARY_DIR}/src/lib/ares_gethostbyaddr.c" + "${LIBRARY_DIR}/src/lib/ares_gethostbyname.c" + "${LIBRARY_DIR}/src/lib/ares_getnameinfo.c" + "${LIBRARY_DIR}/src/lib/ares_getsock.c" + "${LIBRARY_DIR}/src/lib/ares_init.c" + "${LIBRARY_DIR}/src/lib/ares_library_init.c" + "${LIBRARY_DIR}/src/lib/ares_llist.c" + "${LIBRARY_DIR}/src/lib/ares_mkquery.c" + "${LIBRARY_DIR}/src/lib/ares_create_query.c" + "${LIBRARY_DIR}/src/lib/ares_nowarn.c" + "${LIBRARY_DIR}/src/lib/ares_options.c" + "${LIBRARY_DIR}/src/lib/ares_parse_a_reply.c" + "${LIBRARY_DIR}/src/lib/ares_parse_aaaa_reply.c" + "${LIBRARY_DIR}/src/lib/ares_parse_caa_reply.c" + "${LIBRARY_DIR}/src/lib/ares_parse_mx_reply.c" + "${LIBRARY_DIR}/src/lib/ares_parse_naptr_reply.c" + "${LIBRARY_DIR}/src/lib/ares_parse_ns_reply.c" + "${LIBRARY_DIR}/src/lib/ares_parse_ptr_reply.c" + "${LIBRARY_DIR}/src/lib/ares_parse_soa_reply.c" + "${LIBRARY_DIR}/src/lib/ares_parse_srv_reply.c" + "${LIBRARY_DIR}/src/lib/ares_parse_txt_reply.c" + "${LIBRARY_DIR}/src/lib/ares_parse_uri_reply.c" + "${LIBRARY_DIR}/src/lib/ares_platform.c" + "${LIBRARY_DIR}/src/lib/ares_process.c" + "${LIBRARY_DIR}/src/lib/ares_query.c" + "${LIBRARY_DIR}/src/lib/ares_search.c" + "${LIBRARY_DIR}/src/lib/ares_send.c" + "${LIBRARY_DIR}/src/lib/ares_strcasecmp.c" + "${LIBRARY_DIR}/src/lib/ares_strdup.c" + "${LIBRARY_DIR}/src/lib/ares_strerror.c" + "${LIBRARY_DIR}/src/lib/ares_strsplit.c" + "${LIBRARY_DIR}/src/lib/ares_timeout.c" + "${LIBRARY_DIR}/src/lib/ares_version.c" + "${LIBRARY_DIR}/src/lib/ares_writev.c" + "${LIBRARY_DIR}/src/lib/bitncmp.c" + "${LIBRARY_DIR}/src/lib/inet_net_pton.c" + "${LIBRARY_DIR}/src/lib/inet_ntop.c" + "${LIBRARY_DIR}/src/lib/windows_port.c" +) + +if (USE_STATIC_LIBRARIES) + add_library(_c-ares STATIC ${SRCS}) + target_compile_definitions(_c-ares PUBLIC CARES_STATICLIB) +else() + add_library(_c-ares SHARED ${SRCS}) + target_compile_definitions(_c-ares PUBLIC CARES_BUILDING_LIBRARY) endif() -# Force use of c-ares inet_net_pton instead of libresolv one -set(HAVE_INET_NET_PTON OFF CACHE BOOL "" FORCE) +target_compile_definitions(_c-ares PRIVATE HAVE_CONFIG_H=1) -add_subdirectory("../c-ares/" "../c-ares/") +target_include_directories(_c-ares SYSTEM PUBLIC + "${LIBRARY_DIR}/src/lib" + "${LIBRARY_DIR}/include" +) -add_library(ch_contrib::c-ares ALIAS c-ares) \ No newline at end of file +# Platform-specific include directories. The original build system does a lot of checks to eventually generate two header files with defines: +# ares_build.h and ares_config.h. To update, run the original CMake build in c-ares for each platform and copy the headers into the +# platform-specific folder. +# For the platform-specific compile definitions, see c-ares top-level CMakeLists.txt. +if (OS_LINUX) + target_include_directories(_c-ares SYSTEM PUBLIC "${ClickHouse_SOURCE_DIR}/contrib/c-ares-cmake/linux") + target_compile_definitions(_c-ares PRIVATE -D_GNU_SOURCE -D_POSIX_C_SOURCE=199309L -D_XOPEN_SOURCE=600) +elseif (OS_DARWIN) + target_include_directories(_c-ares SYSTEM PUBLIC "${ClickHouse_SOURCE_DIR}/contrib/c-ares-cmake/darwin") + target_compile_definitions(_c-ares PRIVATE -D_DARWIN_C_SOURCE) +elseif (OS_FREEBSD) + target_include_directories(_c-ares SYSTEM PUBLIC "${ClickHouse_SOURCE_DIR}/contrib/c-ares-cmake/freebsd") +endif() + +add_library(ch_contrib::c-ares ALIAS _c-ares) diff --git a/contrib/c-ares-cmake/darwin/ares_build.h b/contrib/c-ares-cmake/darwin/ares_build.h new file mode 100644 index 00000000000..bf7402e7997 --- /dev/null +++ b/contrib/c-ares-cmake/darwin/ares_build.h @@ -0,0 +1,43 @@ +#ifndef __CARES_BUILD_H +#define __CARES_BUILD_H + +#define CARES_TYPEOF_ARES_SOCKLEN_T socklen_t +#define CARES_TYPEOF_ARES_SSIZE_T ssize_t + +/* Prefix names with CARES_ to make sure they don't conflict with other config.h + * files. We need to include some dependent headers that may be system specific + * for C-Ares */ +#define CARES_HAVE_SYS_TYPES_H +#define CARES_HAVE_SYS_SOCKET_H +/* #undef CARES_HAVE_WINDOWS_H */ +/* #undef CARES_HAVE_WS2TCPIP_H */ +/* #undef CARES_HAVE_WINSOCK2_H */ +/* #undef CARES_HAVE_WINDOWS_H */ +#define CARES_HAVE_ARPA_NAMESER_H +#define CARES_HAVE_ARPA_NAMESER_COMPAT_H + +#ifdef CARES_HAVE_SYS_TYPES_H +# include +#endif + +#ifdef CARES_HAVE_SYS_SOCKET_H +# include +#endif + +#ifdef CARES_HAVE_WINSOCK2_H +# include +#endif + +#ifdef CARES_HAVE_WS2TCPIP_H +# include +#endif + +#ifdef CARES_HAVE_WINDOWS_H +# include +#endif + + +typedef CARES_TYPEOF_ARES_SOCKLEN_T ares_socklen_t; +typedef CARES_TYPEOF_ARES_SSIZE_T ares_ssize_t; + +#endif /* __CARES_BUILD_H */ diff --git a/contrib/c-ares-cmake/darwin/ares_config.h b/contrib/c-ares-cmake/darwin/ares_config.h new file mode 100644 index 00000000000..64af3836f3f --- /dev/null +++ b/contrib/c-ares-cmake/darwin/ares_config.h @@ -0,0 +1,432 @@ +/* Generated from ares_config.h.cmake */ + +/* Define if building universal (internal helper macro) */ +#undef AC_APPLE_UNIVERSAL_BUILD + +/* define this if ares is built for a big endian system */ +#undef ARES_BIG_ENDIAN + +/* when building as static part of libcurl */ +#undef BUILDING_LIBCURL + +/* Defined for build that exposes internal static functions for testing. */ +#undef CARES_EXPOSE_STATICS + +/* Defined for build with symbol hiding. */ +#undef CARES_SYMBOL_HIDING + +/* Definition to make a library symbol externally visible. */ +#undef CARES_SYMBOL_SCOPE_EXTERN + +/* Use resolver library to configure cares */ +/* #undef CARES_USE_LIBRESOLV */ + +/* if a /etc/inet dir is being used */ +#undef ETC_INET + +/* Define to the type of arg 2 for gethostname. */ +#define GETHOSTNAME_TYPE_ARG2 size_t + +/* Define to the type qualifier of arg 1 for getnameinfo. */ +#define GETNAMEINFO_QUAL_ARG1 + +/* Define to the type of arg 1 for getnameinfo. */ +#define GETNAMEINFO_TYPE_ARG1 struct sockaddr * + +/* Define to the type of arg 2 for getnameinfo. */ +#define GETNAMEINFO_TYPE_ARG2 socklen_t + +/* Define to the type of args 4 and 6 for getnameinfo. */ +#define GETNAMEINFO_TYPE_ARG46 socklen_t + +/* Define to the type of arg 7 for getnameinfo. */ +#define GETNAMEINFO_TYPE_ARG7 int + +/* Specifies the number of arguments to getservbyport_r */ +#define GETSERVBYPORT_R_ARGS + +/* Specifies the number of arguments to getservbyname_r */ +#define GETSERVBYNAME_R_ARGS + +/* Define to 1 if you have AF_INET6. */ +#define HAVE_AF_INET6 + +/* Define to 1 if you have the header file. */ +#define HAVE_ARPA_INET_H + +/* Define to 1 if you have the header file. */ +#define HAVE_ARPA_NAMESER_COMPAT_H + +/* Define to 1 if you have the header file. */ +#define HAVE_ARPA_NAMESER_H + +/* Define to 1 if you have the header file. */ +#define HAVE_ASSERT_H + +/* Define to 1 if you have the `bitncmp' function. */ +/* #undef HAVE_BITNCMP */ + +/* Define to 1 if bool is an available type. */ +#define HAVE_BOOL_T + +/* Define to 1 if you have the clock_gettime function and monotonic timer. */ +#define HAVE_CLOCK_GETTIME_MONOTONIC + +/* Define to 1 if you have the closesocket function. */ +/* #undef HAVE_CLOSESOCKET */ + +/* Define to 1 if you have the CloseSocket camel case function. */ +/* #undef HAVE_CLOSESOCKET_CAMEL */ + +/* Define to 1 if you have the connect function. */ +#define HAVE_CONNECT + +/* define if the compiler supports basic C++11 syntax */ +/* #undef HAVE_CXX11 */ + +/* Define to 1 if you have the header file. */ +#define HAVE_DLFCN_H + +/* Define to 1 if you have the header file. */ +#define HAVE_ERRNO_H + +/* Define to 1 if you have the fcntl function. */ +#define HAVE_FCNTL + +/* Define to 1 if you have the header file. */ +#define HAVE_FCNTL_H + +/* Define to 1 if you have a working fcntl O_NONBLOCK function. */ +#define HAVE_FCNTL_O_NONBLOCK + +/* Define to 1 if you have the freeaddrinfo function. */ +#define HAVE_FREEADDRINFO + +/* Define to 1 if you have a working getaddrinfo function. */ +#define HAVE_GETADDRINFO + +/* Define to 1 if the getaddrinfo function is threadsafe. */ +#define HAVE_GETADDRINFO_THREADSAFE + +/* Define to 1 if you have the getenv function. */ +#define HAVE_GETENV + +/* Define to 1 if you have the gethostbyaddr function. */ +#define HAVE_GETHOSTBYADDR + +/* Define to 1 if you have the gethostbyname function. */ +#define HAVE_GETHOSTBYNAME + +/* Define to 1 if you have the gethostname function. */ +#define HAVE_GETHOSTNAME + +/* Define to 1 if you have the getnameinfo function. */ +#define HAVE_GETNAMEINFO + +/* Define to 1 if you have the getservbyport_r function. */ +/* #undef HAVE_GETSERVBYPORT_R */ + +/* Define to 1 if you have the getservbyname_r function. */ +/* #undef HAVE_GETSERVBYNAME_R */ + +/* Define to 1 if you have the `gettimeofday' function. */ +#define HAVE_GETTIMEOFDAY + +/* Define to 1 if you have the `if_indextoname' function. */ +#define HAVE_IF_INDEXTONAME + +/* Define to 1 if you have a IPv6 capable working inet_net_pton function. */ +/* #undef HAVE_INET_NET_PTON */ + +/* Define to 1 if you have a IPv6 capable working inet_ntop function. */ +#define HAVE_INET_NTOP + +/* Define to 1 if you have a IPv6 capable working inet_pton function. */ +#define HAVE_INET_PTON + +/* Define to 1 if you have the header file. */ +#define HAVE_INTTYPES_H + +/* Define to 1 if you have the ioctl function. */ +#define HAVE_IOCTL + +/* Define to 1 if you have the ioctlsocket function. */ +/* #undef HAVE_IOCTLSOCKET */ + +/* Define to 1 if you have the IoctlSocket camel case function. */ +/* #undef HAVE_IOCTLSOCKET_CAMEL */ + +/* Define to 1 if you have a working IoctlSocket camel case FIONBIO function. + */ +/* #undef HAVE_IOCTLSOCKET_CAMEL_FIONBIO */ + +/* Define to 1 if you have a working ioctlsocket FIONBIO function. */ +/* #undef HAVE_IOCTLSOCKET_FIONBIO */ + +/* Define to 1 if you have a working ioctl FIONBIO function. */ +#define HAVE_IOCTL_FIONBIO + +/* Define to 1 if you have a working ioctl SIOCGIFADDR function. */ +#define HAVE_IOCTL_SIOCGIFADDR + +/* Define to 1 if you have the `resolve' library (-lresolve). */ +/* #undef HAVE_LIBRESOLV */ + +/* Define to 1 if you have the header file. */ +#define HAVE_LIMITS_H + +/* if your compiler supports LL */ +#define HAVE_LL + +/* Define to 1 if the compiler supports the 'long long' data type. */ +#define HAVE_LONGLONG + +/* Define to 1 if you have the malloc.h header file. */ +/* #undef HAVE_MALLOC_H */ + +/* Define to 1 if you have the memory.h header file. */ +#define HAVE_MEMORY_H + +/* Define to 1 if you have the MSG_NOSIGNAL flag. */ +/* #undef HAVE_MSG_NOSIGNAL */ + +/* Define to 1 if you have the header file. */ +#define HAVE_NETDB_H + +/* Define to 1 if you have the header file. */ +#define HAVE_NETINET_IN_H + +/* Define to 1 if you have the header file. */ +#define HAVE_NETINET_TCP_H + +/* Define to 1 if you have the header file. */ +#define HAVE_NET_IF_H + +/* Define to 1 if you have PF_INET6. */ +#define HAVE_PF_INET6 + +/* Define to 1 if you have the recv function. */ +#define HAVE_RECV + +/* Define to 1 if you have the recvfrom function. */ +#define HAVE_RECVFROM + +/* Define to 1 if you have the send function. */ +#define HAVE_SEND + +/* Define to 1 if you have the setsockopt function. */ +#define HAVE_SETSOCKOPT + +/* Define to 1 if you have a working setsockopt SO_NONBLOCK function. */ +/* #undef HAVE_SETSOCKOPT_SO_NONBLOCK */ + +/* Define to 1 if you have the header file. */ +#define HAVE_SIGNAL_H + +/* Define to 1 if sig_atomic_t is an available typedef. */ +#define HAVE_SIG_ATOMIC_T + +/* Define to 1 if sig_atomic_t is already defined as volatile. */ +/* #undef HAVE_SIG_ATOMIC_T_VOLATILE */ + +/* Define to 1 if your struct sockaddr_in6 has sin6_scope_id. */ +#define HAVE_SOCKADDR_IN6_SIN6_SCOPE_ID + +/* Define to 1 if you have the socket function. */ +#define HAVE_SOCKET + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SOCKET_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_STDBOOL_H + +/* Define to 1 if you have the header file. */ +#define HAVE_STDINT_H + +/* Define to 1 if you have the header file. */ +#define HAVE_STDLIB_H + +/* Define to 1 if you have the strcasecmp function. */ +#define HAVE_STRCASECMP + +/* Define to 1 if you have the strcmpi function. */ +/* #undef HAVE_STRCMPI */ + +/* Define to 1 if you have the strdup function. */ +#define HAVE_STRDUP + +/* Define to 1 if you have the stricmp function. */ +/* #undef HAVE_STRICMP */ + +/* Define to 1 if you have the header file. */ +#define HAVE_STRINGS_H + +/* Define to 1 if you have the header file. */ +#define HAVE_STRING_H + +/* Define to 1 if you have the strncasecmp function. */ +#define HAVE_STRNCASECMP + +/* Define to 1 if you have the strncmpi function. */ +/* #undef HAVE_STRNCMPI */ + +/* Define to 1 if you have the strnicmp function. */ +/* #undef HAVE_STRNICMP */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_STROPTS_H */ + +/* Define to 1 if you have struct addrinfo. */ +#define HAVE_STRUCT_ADDRINFO + +/* Define to 1 if you have struct in6_addr. */ +#define HAVE_STRUCT_IN6_ADDR + +/* Define to 1 if you have struct sockaddr_in6. */ +#define HAVE_STRUCT_SOCKADDR_IN6 + +/* if struct sockaddr_storage is defined */ +#define HAVE_STRUCT_SOCKADDR_STORAGE + +/* Define to 1 if you have the timeval struct. */ +#define HAVE_STRUCT_TIMEVAL + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_IOCTL_H + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_PARAM_H + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_SELECT_H + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_SOCKET_H + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_STAT_H + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_TIME_H + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_TYPES_H + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_UIO_H + +/* Define to 1 if you have the header file. */ +#define HAVE_TIME_H + +/* Define to 1 if you have the header file. */ +#define HAVE_UNISTD_H + +/* Define to 1 if you have the windows.h header file. */ +/* #undef HAVE_WINDOWS_H */ + +/* Define to 1 if you have the winsock2.h header file. */ +/* #undef HAVE_WINSOCK2_H */ + +/* Define to 1 if you have the winsock.h header file. */ +/* #undef HAVE_WINSOCK_H */ + +/* Define to 1 if you have the writev function. */ +#define HAVE_WRITEV + +/* Define to 1 if you have the ws2tcpip.h header file. */ +/* #undef HAVE_WS2TCPIP_H */ + +/* Define to 1 if you have the __system_property_get function */ +#define HAVE___SYSTEM_PROPERTY_GET + +/* Define to 1 if you need the malloc.h header file even with stdlib.h */ +/* #undef NEED_MALLOC_H */ + +/* Define to 1 if you need the memory.h header file even with stdlib.h */ +/* #undef NEED_MEMORY_H */ + +/* a suitable file/device to read random data from */ +#define CARES_RANDOM_FILE "/dev/urandom" + +/* Define to the type qualifier pointed by arg 5 for recvfrom. */ +#define RECVFROM_QUAL_ARG5 + +/* Define to the type of arg 1 for recvfrom. */ +#define RECVFROM_TYPE_ARG1 int + +/* Define to the type pointed by arg 2 for recvfrom. */ +#define RECVFROM_TYPE_ARG2 void * + +/* Define to 1 if the type pointed by arg 2 for recvfrom is void. */ +#define RECVFROM_TYPE_ARG2_IS_VOID 0 + +/* Define to the type of arg 3 for recvfrom. */ +#define RECVFROM_TYPE_ARG3 size_t + +/* Define to the type of arg 4 for recvfrom. */ +#define RECVFROM_TYPE_ARG4 int + +/* Define to the type pointed by arg 5 for recvfrom. */ +#define RECVFROM_TYPE_ARG5 struct sockaddr * + +/* Define to 1 if the type pointed by arg 5 for recvfrom is void. */ +#define RECVFROM_TYPE_ARG5_IS_VOID 0 + +/* Define to the type pointed by arg 6 for recvfrom. */ +#define RECVFROM_TYPE_ARG6 socklen_t * + +/* Define to 1 if the type pointed by arg 6 for recvfrom is void. */ +#define RECVFROM_TYPE_ARG6_IS_VOID 0 + +/* Define to the function return type for recvfrom. */ +#define RECVFROM_TYPE_RETV ssize_t + +/* Define to the type of arg 1 for recv. */ +#define RECV_TYPE_ARG1 int + +/* Define to the type of arg 2 for recv. */ +#define RECV_TYPE_ARG2 void * + +/* Define to the type of arg 3 for recv. */ +#define RECV_TYPE_ARG3 size_t + +/* Define to the type of arg 4 for recv. */ +#define RECV_TYPE_ARG4 int + +/* Define to the function return type for recv. */ +#define RECV_TYPE_RETV ssize_t + +/* Define as the return type of signal handlers (`int' or `void'). */ +#define RETSIGTYPE + +/* Define to the type qualifier of arg 2 for send. */ +#define SEND_QUAL_ARG2 + +/* Define to the type of arg 1 for send. */ +#define SEND_TYPE_ARG1 int + +/* Define to the type of arg 2 for send. */ +#define SEND_TYPE_ARG2 void * + +/* Define to the type of arg 3 for send. */ +#define SEND_TYPE_ARG3 size_t + +/* Define to the type of arg 4 for send. */ +#define SEND_TYPE_ARG4 int + +/* Define to the function return type for send. */ +#define SEND_TYPE_RETV ssize_t + +/* Define to 1 if you can safely include both and . */ +#define TIME_WITH_SYS_TIME + +/* Define to disable non-blocking sockets. */ +#undef USE_BLOCKING_SOCKETS + +/* Define to avoid automatic inclusion of winsock.h */ +#undef WIN32_LEAN_AND_MEAN + +/* Type to use in place of in_addr_t when system does not provide it. */ +#undef in_addr_t + diff --git a/contrib/c-ares-cmake/freebsd/ares_build.h b/contrib/c-ares-cmake/freebsd/ares_build.h new file mode 100644 index 00000000000..bf7402e7997 --- /dev/null +++ b/contrib/c-ares-cmake/freebsd/ares_build.h @@ -0,0 +1,43 @@ +#ifndef __CARES_BUILD_H +#define __CARES_BUILD_H + +#define CARES_TYPEOF_ARES_SOCKLEN_T socklen_t +#define CARES_TYPEOF_ARES_SSIZE_T ssize_t + +/* Prefix names with CARES_ to make sure they don't conflict with other config.h + * files. We need to include some dependent headers that may be system specific + * for C-Ares */ +#define CARES_HAVE_SYS_TYPES_H +#define CARES_HAVE_SYS_SOCKET_H +/* #undef CARES_HAVE_WINDOWS_H */ +/* #undef CARES_HAVE_WS2TCPIP_H */ +/* #undef CARES_HAVE_WINSOCK2_H */ +/* #undef CARES_HAVE_WINDOWS_H */ +#define CARES_HAVE_ARPA_NAMESER_H +#define CARES_HAVE_ARPA_NAMESER_COMPAT_H + +#ifdef CARES_HAVE_SYS_TYPES_H +# include +#endif + +#ifdef CARES_HAVE_SYS_SOCKET_H +# include +#endif + +#ifdef CARES_HAVE_WINSOCK2_H +# include +#endif + +#ifdef CARES_HAVE_WS2TCPIP_H +# include +#endif + +#ifdef CARES_HAVE_WINDOWS_H +# include +#endif + + +typedef CARES_TYPEOF_ARES_SOCKLEN_T ares_socklen_t; +typedef CARES_TYPEOF_ARES_SSIZE_T ares_ssize_t; + +#endif /* __CARES_BUILD_H */ diff --git a/contrib/c-ares-cmake/freebsd/ares_config.h b/contrib/c-ares-cmake/freebsd/ares_config.h new file mode 100644 index 00000000000..a7836e0e802 --- /dev/null +++ b/contrib/c-ares-cmake/freebsd/ares_config.h @@ -0,0 +1,432 @@ +/* Generated from ares_config.h.cmake */ + +/* Define if building universal (internal helper macro) */ +#undef AC_APPLE_UNIVERSAL_BUILD + +/* define this if ares is built for a big endian system */ +#undef ARES_BIG_ENDIAN + +/* when building as static part of libcurl */ +#undef BUILDING_LIBCURL + +/* Defined for build that exposes internal static functions for testing. */ +#undef CARES_EXPOSE_STATICS + +/* Defined for build with symbol hiding. */ +#undef CARES_SYMBOL_HIDING + +/* Definition to make a library symbol externally visible. */ +#undef CARES_SYMBOL_SCOPE_EXTERN + +/* Use resolver library to configure cares */ +/* #undef CARES_USE_LIBRESOLV */ + +/* if a /etc/inet dir is being used */ +#undef ETC_INET + +/* Define to the type of arg 2 for gethostname. */ +#define GETHOSTNAME_TYPE_ARG2 size_t + +/* Define to the type qualifier of arg 1 for getnameinfo. */ +#define GETNAMEINFO_QUAL_ARG1 + +/* Define to the type of arg 1 for getnameinfo. */ +#define GETNAMEINFO_TYPE_ARG1 struct sockaddr * + +/* Define to the type of arg 2 for getnameinfo. */ +#define GETNAMEINFO_TYPE_ARG2 socklen_t + +/* Define to the type of args 4 and 6 for getnameinfo. */ +#define GETNAMEINFO_TYPE_ARG46 socklen_t + +/* Define to the type of arg 7 for getnameinfo. */ +#define GETNAMEINFO_TYPE_ARG7 int + +/* Specifies the number of arguments to getservbyport_r */ +#define GETSERVBYPORT_R_ARGS 6 + +/* Specifies the number of arguments to getservbyname_r */ +#define GETSERVBYNAME_R_ARGS 6 + +/* Define to 1 if you have AF_INET6. */ +#define HAVE_AF_INET6 + +/* Define to 1 if you have the header file. */ +#define HAVE_ARPA_INET_H + +/* Define to 1 if you have the header file. */ +#define HAVE_ARPA_NAMESER_COMPAT_H + +/* Define to 1 if you have the header file. */ +#define HAVE_ARPA_NAMESER_H + +/* Define to 1 if you have the header file. */ +#define HAVE_ASSERT_H + +/* Define to 1 if you have the `bitncmp' function. */ +/* #undef HAVE_BITNCMP */ + +/* Define to 1 if bool is an available type. */ +#define HAVE_BOOL_T + +/* Define to 1 if you have the clock_gettime function and monotonic timer. */ +#define HAVE_CLOCK_GETTIME_MONOTONIC + +/* Define to 1 if you have the closesocket function. */ +/* #undef HAVE_CLOSESOCKET */ + +/* Define to 1 if you have the CloseSocket camel case function. */ +/* #undef HAVE_CLOSESOCKET_CAMEL */ + +/* Define to 1 if you have the connect function. */ +#define HAVE_CONNECT + +/* define if the compiler supports basic C++11 syntax */ +/* #undef HAVE_CXX11 */ + +/* Define to 1 if you have the header file. */ +#define HAVE_DLFCN_H + +/* Define to 1 if you have the header file. */ +#define HAVE_ERRNO_H + +/* Define to 1 if you have the fcntl function. */ +#define HAVE_FCNTL + +/* Define to 1 if you have the header file. */ +#define HAVE_FCNTL_H + +/* Define to 1 if you have a working fcntl O_NONBLOCK function. */ +#define HAVE_FCNTL_O_NONBLOCK + +/* Define to 1 if you have the freeaddrinfo function. */ +#define HAVE_FREEADDRINFO + +/* Define to 1 if you have a working getaddrinfo function. */ +#define HAVE_GETADDRINFO + +/* Define to 1 if the getaddrinfo function is threadsafe. */ +#define HAVE_GETADDRINFO_THREADSAFE + +/* Define to 1 if you have the getenv function. */ +#define HAVE_GETENV + +/* Define to 1 if you have the gethostbyaddr function. */ +#define HAVE_GETHOSTBYADDR + +/* Define to 1 if you have the gethostbyname function. */ +#define HAVE_GETHOSTBYNAME + +/* Define to 1 if you have the gethostname function. */ +#define HAVE_GETHOSTNAME + +/* Define to 1 if you have the getnameinfo function. */ +#define HAVE_GETNAMEINFO + +/* Define to 1 if you have the getservbyport_r function. */ +#define HAVE_GETSERVBYPORT_R + +/* Define to 1 if you have the getservbyname_r function. */ +#define HAVE_GETSERVBYNAME_R + +/* Define to 1 if you have the `gettimeofday' function. */ +#define HAVE_GETTIMEOFDAY + +/* Define to 1 if you have the `if_indextoname' function. */ +#define HAVE_IF_INDEXTONAME + +/* Define to 1 if you have a IPv6 capable working inet_net_pton function. */ +/* #undef HAVE_INET_NET_PTON */ + +/* Define to 1 if you have a IPv6 capable working inet_ntop function. */ +#define HAVE_INET_NTOP + +/* Define to 1 if you have a IPv6 capable working inet_pton function. */ +#define HAVE_INET_PTON + +/* Define to 1 if you have the header file. */ +#define HAVE_INTTYPES_H + +/* Define to 1 if you have the ioctl function. */ +#define HAVE_IOCTL + +/* Define to 1 if you have the ioctlsocket function. */ +/* #undef HAVE_IOCTLSOCKET */ + +/* Define to 1 if you have the IoctlSocket camel case function. */ +/* #undef HAVE_IOCTLSOCKET_CAMEL */ + +/* Define to 1 if you have a working IoctlSocket camel case FIONBIO function. + */ +/* #undef HAVE_IOCTLSOCKET_CAMEL_FIONBIO */ + +/* Define to 1 if you have a working ioctlsocket FIONBIO function. */ +/* #undef HAVE_IOCTLSOCKET_FIONBIO */ + +/* Define to 1 if you have a working ioctl FIONBIO function. */ +#define HAVE_IOCTL_FIONBIO + +/* Define to 1 if you have a working ioctl SIOCGIFADDR function. */ +#define HAVE_IOCTL_SIOCGIFADDR + +/* Define to 1 if you have the `resolve' library (-lresolve). */ +/* #undef HAVE_LIBRESOLV */ + +/* Define to 1 if you have the header file. */ +#define HAVE_LIMITS_H + +/* if your compiler supports LL */ +#define HAVE_LL + +/* Define to 1 if the compiler supports the 'long long' data type. */ +#define HAVE_LONGLONG + +/* Define to 1 if you have the malloc.h header file. */ +/* #undef HAVE_MALLOC_H */ + +/* Define to 1 if you have the memory.h header file. */ +#define HAVE_MEMORY_H + +/* Define to 1 if you have the MSG_NOSIGNAL flag. */ +#define HAVE_MSG_NOSIGNAL + +/* Define to 1 if you have the header file. */ +#define HAVE_NETDB_H + +/* Define to 1 if you have the header file. */ +#define HAVE_NETINET_IN_H + +/* Define to 1 if you have the header file. */ +#define HAVE_NETINET_TCP_H + +/* Define to 1 if you have the header file. */ +#define HAVE_NET_IF_H + +/* Define to 1 if you have PF_INET6. */ +#define HAVE_PF_INET6 + +/* Define to 1 if you have the recv function. */ +#define HAVE_RECV + +/* Define to 1 if you have the recvfrom function. */ +#define HAVE_RECVFROM + +/* Define to 1 if you have the send function. */ +#define HAVE_SEND + +/* Define to 1 if you have the setsockopt function. */ +#define HAVE_SETSOCKOPT + +/* Define to 1 if you have a working setsockopt SO_NONBLOCK function. */ +/* #undef HAVE_SETSOCKOPT_SO_NONBLOCK */ + +/* Define to 1 if you have the header file. */ +#define HAVE_SIGNAL_H + +/* Define to 1 if sig_atomic_t is an available typedef. */ +#define HAVE_SIG_ATOMIC_T + +/* Define to 1 if sig_atomic_t is already defined as volatile. */ +/* #undef HAVE_SIG_ATOMIC_T_VOLATILE */ + +/* Define to 1 if your struct sockaddr_in6 has sin6_scope_id. */ +#define HAVE_SOCKADDR_IN6_SIN6_SCOPE_ID + +/* Define to 1 if you have the socket function. */ +#define HAVE_SOCKET + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SOCKET_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_STDBOOL_H + +/* Define to 1 if you have the header file. */ +#define HAVE_STDINT_H + +/* Define to 1 if you have the header file. */ +#define HAVE_STDLIB_H + +/* Define to 1 if you have the strcasecmp function. */ +#define HAVE_STRCASECMP + +/* Define to 1 if you have the strcmpi function. */ +/* #undef HAVE_STRCMPI */ + +/* Define to 1 if you have the strdup function. */ +#define HAVE_STRDUP + +/* Define to 1 if you have the stricmp function. */ +/* #undef HAVE_STRICMP */ + +/* Define to 1 if you have the header file. */ +#define HAVE_STRINGS_H + +/* Define to 1 if you have the header file. */ +#define HAVE_STRING_H + +/* Define to 1 if you have the strncasecmp function. */ +#define HAVE_STRNCASECMP + +/* Define to 1 if you have the strncmpi function. */ +/* #undef HAVE_STRNCMPI */ + +/* Define to 1 if you have the strnicmp function. */ +/* #undef HAVE_STRNICMP */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_STROPTS_H */ + +/* Define to 1 if you have struct addrinfo. */ +#define HAVE_STRUCT_ADDRINFO + +/* Define to 1 if you have struct in6_addr. */ +#define HAVE_STRUCT_IN6_ADDR + +/* Define to 1 if you have struct sockaddr_in6. */ +#define HAVE_STRUCT_SOCKADDR_IN6 + +/* if struct sockaddr_storage is defined */ +#define HAVE_STRUCT_SOCKADDR_STORAGE + +/* Define to 1 if you have the timeval struct. */ +#define HAVE_STRUCT_TIMEVAL + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_IOCTL_H + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_PARAM_H + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_SELECT_H + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_SOCKET_H + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_STAT_H + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_TIME_H + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_TYPES_H + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_UIO_H + +/* Define to 1 if you have the header file. */ +#define HAVE_TIME_H + +/* Define to 1 if you have the header file. */ +#define HAVE_UNISTD_H + +/* Define to 1 if you have the windows.h header file. */ +/* #undef HAVE_WINDOWS_H */ + +/* Define to 1 if you have the winsock2.h header file. */ +/* #undef HAVE_WINSOCK2_H */ + +/* Define to 1 if you have the winsock.h header file. */ +/* #undef HAVE_WINSOCK_H */ + +/* Define to 1 if you have the writev function. */ +#define HAVE_WRITEV + +/* Define to 1 if you have the ws2tcpip.h header file. */ +/* #undef HAVE_WS2TCPIP_H */ + +/* Define to 1 if you have the __system_property_get function */ +#define HAVE___SYSTEM_PROPERTY_GET + +/* Define to 1 if you need the malloc.h header file even with stdlib.h */ +/* #undef NEED_MALLOC_H */ + +/* Define to 1 if you need the memory.h header file even with stdlib.h */ +/* #undef NEED_MEMORY_H */ + +/* a suitable file/device to read random data from */ +#define CARES_RANDOM_FILE "/dev/urandom" + +/* Define to the type qualifier pointed by arg 5 for recvfrom. */ +#define RECVFROM_QUAL_ARG5 + +/* Define to the type of arg 1 for recvfrom. */ +#define RECVFROM_TYPE_ARG1 int + +/* Define to the type pointed by arg 2 for recvfrom. */ +#define RECVFROM_TYPE_ARG2 void * + +/* Define to 1 if the type pointed by arg 2 for recvfrom is void. */ +#define RECVFROM_TYPE_ARG2_IS_VOID 0 + +/* Define to the type of arg 3 for recvfrom. */ +#define RECVFROM_TYPE_ARG3 size_t + +/* Define to the type of arg 4 for recvfrom. */ +#define RECVFROM_TYPE_ARG4 int + +/* Define to the type pointed by arg 5 for recvfrom. */ +#define RECVFROM_TYPE_ARG5 struct sockaddr * + +/* Define to 1 if the type pointed by arg 5 for recvfrom is void. */ +#define RECVFROM_TYPE_ARG5_IS_VOID 0 + +/* Define to the type pointed by arg 6 for recvfrom. */ +#define RECVFROM_TYPE_ARG6 socklen_t * + +/* Define to 1 if the type pointed by arg 6 for recvfrom is void. */ +#define RECVFROM_TYPE_ARG6_IS_VOID 0 + +/* Define to the function return type for recvfrom. */ +#define RECVFROM_TYPE_RETV ssize_t + +/* Define to the type of arg 1 for recv. */ +#define RECV_TYPE_ARG1 int + +/* Define to the type of arg 2 for recv. */ +#define RECV_TYPE_ARG2 void * + +/* Define to the type of arg 3 for recv. */ +#define RECV_TYPE_ARG3 size_t + +/* Define to the type of arg 4 for recv. */ +#define RECV_TYPE_ARG4 int + +/* Define to the function return type for recv. */ +#define RECV_TYPE_RETV ssize_t + +/* Define as the return type of signal handlers (`int' or `void'). */ +#define RETSIGTYPE + +/* Define to the type qualifier of arg 2 for send. */ +#define SEND_QUAL_ARG2 + +/* Define to the type of arg 1 for send. */ +#define SEND_TYPE_ARG1 int + +/* Define to the type of arg 2 for send. */ +#define SEND_TYPE_ARG2 void * + +/* Define to the type of arg 3 for send. */ +#define SEND_TYPE_ARG3 size_t + +/* Define to the type of arg 4 for send. */ +#define SEND_TYPE_ARG4 int + +/* Define to the function return type for send. */ +#define SEND_TYPE_RETV ssize_t + +/* Define to 1 if you can safely include both and . */ +#define TIME_WITH_SYS_TIME + +/* Define to disable non-blocking sockets. */ +#undef USE_BLOCKING_SOCKETS + +/* Define to avoid automatic inclusion of winsock.h */ +#undef WIN32_LEAN_AND_MEAN + +/* Type to use in place of in_addr_t when system does not provide it. */ +#undef in_addr_t + diff --git a/contrib/c-ares-cmake/linux/ares_build.h b/contrib/c-ares-cmake/linux/ares_build.h new file mode 100644 index 00000000000..bf7402e7997 --- /dev/null +++ b/contrib/c-ares-cmake/linux/ares_build.h @@ -0,0 +1,43 @@ +#ifndef __CARES_BUILD_H +#define __CARES_BUILD_H + +#define CARES_TYPEOF_ARES_SOCKLEN_T socklen_t +#define CARES_TYPEOF_ARES_SSIZE_T ssize_t + +/* Prefix names with CARES_ to make sure they don't conflict with other config.h + * files. We need to include some dependent headers that may be system specific + * for C-Ares */ +#define CARES_HAVE_SYS_TYPES_H +#define CARES_HAVE_SYS_SOCKET_H +/* #undef CARES_HAVE_WINDOWS_H */ +/* #undef CARES_HAVE_WS2TCPIP_H */ +/* #undef CARES_HAVE_WINSOCK2_H */ +/* #undef CARES_HAVE_WINDOWS_H */ +#define CARES_HAVE_ARPA_NAMESER_H +#define CARES_HAVE_ARPA_NAMESER_COMPAT_H + +#ifdef CARES_HAVE_SYS_TYPES_H +# include +#endif + +#ifdef CARES_HAVE_SYS_SOCKET_H +# include +#endif + +#ifdef CARES_HAVE_WINSOCK2_H +# include +#endif + +#ifdef CARES_HAVE_WS2TCPIP_H +# include +#endif + +#ifdef CARES_HAVE_WINDOWS_H +# include +#endif + + +typedef CARES_TYPEOF_ARES_SOCKLEN_T ares_socklen_t; +typedef CARES_TYPEOF_ARES_SSIZE_T ares_ssize_t; + +#endif /* __CARES_BUILD_H */ diff --git a/contrib/c-ares-cmake/linux/ares_config.h b/contrib/c-ares-cmake/linux/ares_config.h new file mode 100644 index 00000000000..e0ebf86e842 --- /dev/null +++ b/contrib/c-ares-cmake/linux/ares_config.h @@ -0,0 +1,432 @@ +/* Generated from ares_config.h.cmake */ + +/* Define if building universal (internal helper macro) */ +#undef AC_APPLE_UNIVERSAL_BUILD + +/* define this if ares is built for a big endian system */ +#undef ARES_BIG_ENDIAN + +/* when building as static part of libcurl */ +#undef BUILDING_LIBCURL + +/* Defined for build that exposes internal static functions for testing. */ +#undef CARES_EXPOSE_STATICS + +/* Defined for build with symbol hiding. */ +#undef CARES_SYMBOL_HIDING + +/* Definition to make a library symbol externally visible. */ +#undef CARES_SYMBOL_SCOPE_EXTERN + +/* Use resolver library to configure cares */ +/* #undef CARES_USE_LIBRESOLV */ + +/* if a /etc/inet dir is being used */ +#undef ETC_INET + +/* Define to the type of arg 2 for gethostname. */ +#define GETHOSTNAME_TYPE_ARG2 size_t + +/* Define to the type qualifier of arg 1 for getnameinfo. */ +#define GETNAMEINFO_QUAL_ARG1 + +/* Define to the type of arg 1 for getnameinfo. */ +#define GETNAMEINFO_TYPE_ARG1 struct sockaddr * + +/* Define to the type of arg 2 for getnameinfo. */ +#define GETNAMEINFO_TYPE_ARG2 socklen_t + +/* Define to the type of args 4 and 6 for getnameinfo. */ +#define GETNAMEINFO_TYPE_ARG46 socklen_t + +/* Define to the type of arg 7 for getnameinfo. */ +#define GETNAMEINFO_TYPE_ARG7 int + +/* Specifies the number of arguments to getservbyport_r */ +#define GETSERVBYPORT_R_ARGS 6 + +/* Specifies the number of arguments to getservbyname_r */ +#define GETSERVBYNAME_R_ARGS 6 + +/* Define to 1 if you have AF_INET6. */ +#define HAVE_AF_INET6 + +/* Define to 1 if you have the header file. */ +#define HAVE_ARPA_INET_H + +/* Define to 1 if you have the header file. */ +#define HAVE_ARPA_NAMESER_COMPAT_H + +/* Define to 1 if you have the header file. */ +#define HAVE_ARPA_NAMESER_H + +/* Define to 1 if you have the header file. */ +#define HAVE_ASSERT_H + +/* Define to 1 if you have the `bitncmp' function. */ +/* #undef HAVE_BITNCMP */ + +/* Define to 1 if bool is an available type. */ +#define HAVE_BOOL_T + +/* Define to 1 if you have the clock_gettime function and monotonic timer. */ +#define HAVE_CLOCK_GETTIME_MONOTONIC + +/* Define to 1 if you have the closesocket function. */ +/* #undef HAVE_CLOSESOCKET */ + +/* Define to 1 if you have the CloseSocket camel case function. */ +/* #undef HAVE_CLOSESOCKET_CAMEL */ + +/* Define to 1 if you have the connect function. */ +#define HAVE_CONNECT + +/* define if the compiler supports basic C++11 syntax */ +/* #undef HAVE_CXX11 */ + +/* Define to 1 if you have the header file. */ +#define HAVE_DLFCN_H + +/* Define to 1 if you have the header file. */ +#define HAVE_ERRNO_H + +/* Define to 1 if you have the fcntl function. */ +#define HAVE_FCNTL + +/* Define to 1 if you have the header file. */ +#define HAVE_FCNTL_H + +/* Define to 1 if you have a working fcntl O_NONBLOCK function. */ +#define HAVE_FCNTL_O_NONBLOCK + +/* Define to 1 if you have the freeaddrinfo function. */ +#define HAVE_FREEADDRINFO + +/* Define to 1 if you have a working getaddrinfo function. */ +#define HAVE_GETADDRINFO + +/* Define to 1 if the getaddrinfo function is threadsafe. */ +/* #undef HAVE_GETADDRINFO_THREADSAFE */ + +/* Define to 1 if you have the getenv function. */ +#define HAVE_GETENV + +/* Define to 1 if you have the gethostbyaddr function. */ +#define HAVE_GETHOSTBYADDR + +/* Define to 1 if you have the gethostbyname function. */ +#define HAVE_GETHOSTBYNAME + +/* Define to 1 if you have the gethostname function. */ +#define HAVE_GETHOSTNAME + +/* Define to 1 if you have the getnameinfo function. */ +#define HAVE_GETNAMEINFO + +/* Define to 1 if you have the getservbyport_r function. */ +#define HAVE_GETSERVBYPORT_R + +/* Define to 1 if you have the getservbyname_r function. */ +#define HAVE_GETSERVBYNAME_R + +/* Define to 1 if you have the `gettimeofday' function. */ +#define HAVE_GETTIMEOFDAY + +/* Define to 1 if you have the `if_indextoname' function. */ +#define HAVE_IF_INDEXTONAME + +/* Define to 1 if you have a IPv6 capable working inet_net_pton function. */ +/* #undef HAVE_INET_NET_PTON */ + +/* Define to 1 if you have a IPv6 capable working inet_ntop function. */ +#define HAVE_INET_NTOP + +/* Define to 1 if you have a IPv6 capable working inet_pton function. */ +#define HAVE_INET_PTON + +/* Define to 1 if you have the header file. */ +#define HAVE_INTTYPES_H + +/* Define to 1 if you have the ioctl function. */ +#define HAVE_IOCTL + +/* Define to 1 if you have the ioctlsocket function. */ +/* #undef HAVE_IOCTLSOCKET */ + +/* Define to 1 if you have the IoctlSocket camel case function. */ +/* #undef HAVE_IOCTLSOCKET_CAMEL */ + +/* Define to 1 if you have a working IoctlSocket camel case FIONBIO function. + */ +/* #undef HAVE_IOCTLSOCKET_CAMEL_FIONBIO */ + +/* Define to 1 if you have a working ioctlsocket FIONBIO function. */ +/* #undef HAVE_IOCTLSOCKET_FIONBIO */ + +/* Define to 1 if you have a working ioctl FIONBIO function. */ +#define HAVE_IOCTL_FIONBIO + +/* Define to 1 if you have a working ioctl SIOCGIFADDR function. */ +#define HAVE_IOCTL_SIOCGIFADDR + +/* Define to 1 if you have the `resolve' library (-lresolve). */ +/* #undef HAVE_LIBRESOLV */ + +/* Define to 1 if you have the header file. */ +#define HAVE_LIMITS_H + +/* if your compiler supports LL */ +#define HAVE_LL + +/* Define to 1 if the compiler supports the 'long long' data type. */ +#define HAVE_LONGLONG + +/* Define to 1 if you have the malloc.h header file. */ +#define HAVE_MALLOC_H + +/* Define to 1 if you have the memory.h header file. */ +#define HAVE_MEMORY_H + +/* Define to 1 if you have the MSG_NOSIGNAL flag. */ +#define HAVE_MSG_NOSIGNAL + +/* Define to 1 if you have the header file. */ +#define HAVE_NETDB_H + +/* Define to 1 if you have the header file. */ +#define HAVE_NETINET_IN_H + +/* Define to 1 if you have the header file. */ +#define HAVE_NETINET_TCP_H + +/* Define to 1 if you have the header file. */ +#define HAVE_NET_IF_H + +/* Define to 1 if you have PF_INET6. */ +#define HAVE_PF_INET6 + +/* Define to 1 if you have the recv function. */ +#define HAVE_RECV + +/* Define to 1 if you have the recvfrom function. */ +#define HAVE_RECVFROM + +/* Define to 1 if you have the send function. */ +#define HAVE_SEND + +/* Define to 1 if you have the setsockopt function. */ +#define HAVE_SETSOCKOPT + +/* Define to 1 if you have a working setsockopt SO_NONBLOCK function. */ +/* #undef HAVE_SETSOCKOPT_SO_NONBLOCK */ + +/* Define to 1 if you have the header file. */ +#define HAVE_SIGNAL_H + +/* Define to 1 if sig_atomic_t is an available typedef. */ +#define HAVE_SIG_ATOMIC_T + +/* Define to 1 if sig_atomic_t is already defined as volatile. */ +/* #undef HAVE_SIG_ATOMIC_T_VOLATILE */ + +/* Define to 1 if your struct sockaddr_in6 has sin6_scope_id. */ +#define HAVE_SOCKADDR_IN6_SIN6_SCOPE_ID + +/* Define to 1 if you have the socket function. */ +#define HAVE_SOCKET + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SOCKET_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_STDBOOL_H + +/* Define to 1 if you have the header file. */ +#define HAVE_STDINT_H + +/* Define to 1 if you have the header file. */ +#define HAVE_STDLIB_H + +/* Define to 1 if you have the strcasecmp function. */ +#define HAVE_STRCASECMP + +/* Define to 1 if you have the strcmpi function. */ +/* #undef HAVE_STRCMPI */ + +/* Define to 1 if you have the strdup function. */ +#define HAVE_STRDUP + +/* Define to 1 if you have the stricmp function. */ +/* #undef HAVE_STRICMP */ + +/* Define to 1 if you have the header file. */ +#define HAVE_STRINGS_H + +/* Define to 1 if you have the header file. */ +#define HAVE_STRING_H + +/* Define to 1 if you have the strncasecmp function. */ +#define HAVE_STRNCASECMP + +/* Define to 1 if you have the strncmpi function. */ +/* #undef HAVE_STRNCMPI */ + +/* Define to 1 if you have the strnicmp function. */ +/* #undef HAVE_STRNICMP */ + +/* Define to 1 if you have the header file. */ +#define HAVE_STROPTS_H + +/* Define to 1 if you have struct addrinfo. */ +#define HAVE_STRUCT_ADDRINFO + +/* Define to 1 if you have struct in6_addr. */ +#define HAVE_STRUCT_IN6_ADDR + +/* Define to 1 if you have struct sockaddr_in6. */ +#define HAVE_STRUCT_SOCKADDR_IN6 + +/* if struct sockaddr_storage is defined */ +#define HAVE_STRUCT_SOCKADDR_STORAGE + +/* Define to 1 if you have the timeval struct. */ +#define HAVE_STRUCT_TIMEVAL + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_IOCTL_H + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_PARAM_H + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_SELECT_H + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_SOCKET_H + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_STAT_H + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_TIME_H + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_TYPES_H + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_UIO_H + +/* Define to 1 if you have the header file. */ +#define HAVE_TIME_H + +/* Define to 1 if you have the header file. */ +#define HAVE_UNISTD_H + +/* Define to 1 if you have the windows.h header file. */ +/* #undef HAVE_WINDOWS_H */ + +/* Define to 1 if you have the winsock2.h header file. */ +/* #undef HAVE_WINSOCK2_H */ + +/* Define to 1 if you have the winsock.h header file. */ +/* #undef HAVE_WINSOCK_H */ + +/* Define to 1 if you have the writev function. */ +#define HAVE_WRITEV + +/* Define to 1 if you have the ws2tcpip.h header file. */ +/* #undef HAVE_WS2TCPIP_H */ + +/* Define to 1 if you have the __system_property_get function */ +#define HAVE___SYSTEM_PROPERTY_GET + +/* Define to 1 if you need the malloc.h header file even with stdlib.h */ +/* #undef NEED_MALLOC_H */ + +/* Define to 1 if you need the memory.h header file even with stdlib.h */ +/* #undef NEED_MEMORY_H */ + +/* a suitable file/device to read random data from */ +#define CARES_RANDOM_FILE "/dev/urandom" + +/* Define to the type qualifier pointed by arg 5 for recvfrom. */ +#define RECVFROM_QUAL_ARG5 + +/* Define to the type of arg 1 for recvfrom. */ +#define RECVFROM_TYPE_ARG1 int + +/* Define to the type pointed by arg 2 for recvfrom. */ +#define RECVFROM_TYPE_ARG2 void * + +/* Define to 1 if the type pointed by arg 2 for recvfrom is void. */ +#define RECVFROM_TYPE_ARG2_IS_VOID 0 + +/* Define to the type of arg 3 for recvfrom. */ +#define RECVFROM_TYPE_ARG3 size_t + +/* Define to the type of arg 4 for recvfrom. */ +#define RECVFROM_TYPE_ARG4 int + +/* Define to the type pointed by arg 5 for recvfrom. */ +#define RECVFROM_TYPE_ARG5 struct sockaddr * + +/* Define to 1 if the type pointed by arg 5 for recvfrom is void. */ +#define RECVFROM_TYPE_ARG5_IS_VOID 0 + +/* Define to the type pointed by arg 6 for recvfrom. */ +#define RECVFROM_TYPE_ARG6 socklen_t * + +/* Define to 1 if the type pointed by arg 6 for recvfrom is void. */ +#define RECVFROM_TYPE_ARG6_IS_VOID 0 + +/* Define to the function return type for recvfrom. */ +#define RECVFROM_TYPE_RETV ssize_t + +/* Define to the type of arg 1 for recv. */ +#define RECV_TYPE_ARG1 int + +/* Define to the type of arg 2 for recv. */ +#define RECV_TYPE_ARG2 void * + +/* Define to the type of arg 3 for recv. */ +#define RECV_TYPE_ARG3 size_t + +/* Define to the type of arg 4 for recv. */ +#define RECV_TYPE_ARG4 int + +/* Define to the function return type for recv. */ +#define RECV_TYPE_RETV ssize_t + +/* Define as the return type of signal handlers (`int' or `void'). */ +#define RETSIGTYPE + +/* Define to the type qualifier of arg 2 for send. */ +#define SEND_QUAL_ARG2 + +/* Define to the type of arg 1 for send. */ +#define SEND_TYPE_ARG1 int + +/* Define to the type of arg 2 for send. */ +#define SEND_TYPE_ARG2 void * + +/* Define to the type of arg 3 for send. */ +#define SEND_TYPE_ARG3 size_t + +/* Define to the type of arg 4 for send. */ +#define SEND_TYPE_ARG4 int + +/* Define to the function return type for send. */ +#define SEND_TYPE_RETV ssize_t + +/* Define to 1 if you can safely include both and . */ +#define TIME_WITH_SYS_TIME + +/* Define to disable non-blocking sockets. */ +#undef USE_BLOCKING_SOCKETS + +/* Define to avoid automatic inclusion of winsock.h */ +#undef WIN32_LEAN_AND_MEAN + +/* Type to use in place of in_addr_t when system does not provide it. */ +#undef in_addr_t + diff --git a/contrib/jemalloc-cmake/include_linux_x86_64_musl/jemalloc/internal/jemalloc_internal_defs.h.in b/contrib/jemalloc-cmake/include_linux_x86_64_musl/jemalloc/internal/jemalloc_internal_defs.h.in index ff97d297d8f..e08a2bed2ec 100644 --- a/contrib/jemalloc-cmake/include_linux_x86_64_musl/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/contrib/jemalloc-cmake/include_linux_x86_64_musl/jemalloc/internal/jemalloc_internal_defs.h.in @@ -415,7 +415,7 @@ /* * Defined if strerror_r returns char * if _GNU_SOURCE is defined. */ -#define JEMALLOC_STRERROR_R_RETURNS_CHAR_WITH_GNU_SOURCE +/* #undef JEMALLOC_STRERROR_R_RETURNS_CHAR_WITH_GNU_SOURCE */ /* Performs additional safety checks when defined. */ /* #undef JEMALLOC_OPT_SAFETY_CHECKS */ diff --git a/contrib/krb5 b/contrib/krb5 index d879821c7a4..b89e20367b0 160000 --- a/contrib/krb5 +++ b/contrib/krb5 @@ -1 +1 @@ -Subproject commit d879821c7a4c70b0c3ad739d9951d1a2b1903df7 +Subproject commit b89e20367b074bd02dd118a6534099b21e88b3c3 diff --git a/contrib/krb5-cmake/autoconf_linux.h b/contrib/krb5-cmake/autoconf_linux.h index 7b71d962d9a..54951f866a5 100644 --- a/contrib/krb5-cmake/autoconf_linux.h +++ b/contrib/krb5-cmake/autoconf_linux.h @@ -440,7 +440,9 @@ #define HAVE_STRERROR 1 /* Define to 1 if you have the `strerror_r' function. */ +#ifndef USE_MUSL #define HAVE_STRERROR_R 1 +#endif /* Define to 1 if you have the header file. */ #define HAVE_STRINGS_H 1 diff --git a/contrib/libcpuid b/contrib/libcpuid index 8db3b8d2d32..503083acb77 160000 --- a/contrib/libcpuid +++ b/contrib/libcpuid @@ -1 +1 @@ -Subproject commit 8db3b8d2d32d22437f063ce692a1b9bb15e42d18 +Subproject commit 503083acb77edf9fbce22a05826307dff2ce96e6 diff --git a/contrib/libpq-cmake/CMakeLists.txt b/contrib/libpq-cmake/CMakeLists.txt index 280c0381393..91326422b43 100644 --- a/contrib/libpq-cmake/CMakeLists.txt +++ b/contrib/libpq-cmake/CMakeLists.txt @@ -63,6 +63,13 @@ target_include_directories (_libpq SYSTEM PUBLIC ${LIBPQ_SOURCE_DIR}) target_include_directories (_libpq SYSTEM PUBLIC "${LIBPQ_SOURCE_DIR}/include") target_include_directories (_libpq SYSTEM PRIVATE "${LIBPQ_SOURCE_DIR}/configs") +# NOTE: this is a dirty hack to avoid and instead pg_config.h should be shipped +# for different OS'es like for jemalloc, not one generic for all OS'es like +# now. +if (OS_DARWIN OR OS_FREEBSD OR USE_MUSL) + target_compile_definitions(_libpq PRIVATE -DSTRERROR_R_INT=1) +endif() + target_link_libraries (_libpq PRIVATE OpenSSL::SSL) add_library(ch_contrib::libpq ALIAS _libpq) diff --git a/contrib/librdkafka b/contrib/librdkafka index ff32b4e9eea..6f3b483426a 160000 --- a/contrib/librdkafka +++ b/contrib/librdkafka @@ -1 +1 @@ -Subproject commit ff32b4e9eeafd0b276f010ee969179e4e9e6d0b2 +Subproject commit 6f3b483426a8c8ec950e27e446bec175cf8b553f diff --git a/contrib/llvm b/contrib/llvm index 20607e61728..0db5bf5bd24 160000 --- a/contrib/llvm +++ b/contrib/llvm @@ -1 +1 @@ -Subproject commit 20607e61728e97c969e536644c3c0c1bb1a50672 +Subproject commit 0db5bf5bd2452cd8f1283a1fcdc04845af705bfc diff --git a/contrib/replxx b/contrib/replxx index 3fd0e3c9364..5d04501f93a 160000 --- a/contrib/replxx +++ b/contrib/replxx @@ -1 +1 @@ -Subproject commit 3fd0e3c9364a589447453d9906d854ebd8d385c5 +Subproject commit 5d04501f93a4fb7f0bb8b73b8f614bc986f9e25b diff --git a/contrib/sentry-native b/contrib/sentry-native index f431047ac8d..ae10fb8c224 160000 --- a/contrib/sentry-native +++ b/contrib/sentry-native @@ -1 +1 @@ -Subproject commit f431047ac8da13179c488018dddf1c0d0771a997 +Subproject commit ae10fb8c224c3f41571446e1ed7fd57b9e5e366b diff --git a/contrib/vectorscan b/contrib/vectorscan index 73695e419c2..f6250ae3e5a 160000 --- a/contrib/vectorscan +++ b/contrib/vectorscan @@ -1 +1 @@ -Subproject commit 73695e419c27af7fe2a099c7aa57931cc02aea5d +Subproject commit f6250ae3e5a3085000239313ad0689cc1e00cdc2 diff --git a/contrib/vectorscan-cmake/CMakeLists.txt b/contrib/vectorscan-cmake/CMakeLists.txt index 828f2a17df2..d6c626c1612 100644 --- a/contrib/vectorscan-cmake/CMakeLists.txt +++ b/contrib/vectorscan-cmake/CMakeLists.txt @@ -304,7 +304,7 @@ target_include_directories (_vectorscan SYSTEM PUBLIC "${LIBRARY_DIR}/src") # Please regenerate these files if you update vectorscan. if (ARCH_AMD64) - target_include_directories (_vectorscan PRIVATE x86_64) + target_include_directories (_vectorscan PRIVATE amd64) endif () if (ARCH_AARCH64) diff --git a/contrib/vectorscan-cmake/x86_64/config.h b/contrib/vectorscan-cmake/amd64/config.h similarity index 100% rename from contrib/vectorscan-cmake/x86_64/config.h rename to contrib/vectorscan-cmake/amd64/config.h diff --git a/contrib/zlib-ng-cmake/CMakeLists.txt b/contrib/zlib-ng-cmake/CMakeLists.txt index 371a07dd31a..aa067ba37e0 100644 --- a/contrib/zlib-ng-cmake/CMakeLists.txt +++ b/contrib/zlib-ng-cmake/CMakeLists.txt @@ -2,8 +2,10 @@ set (SOURCE_DIR ${CMAKE_SOURCE_DIR}/contrib/zlib-ng) add_definitions(-DZLIB_COMPAT) add_definitions(-DWITH_GZFILEOP) -add_definitions(-DUNALIGNED_OK) -add_definitions(-DUNALIGNED64_OK) +if(NOT ARCH_S390X) + add_definitions(-DUNALIGNED_OK) + add_definitions(-DUNALIGNED64_OK) +endif() set (HAVE_UNISTD_H 1) add_definitions(-D_LARGEFILE64_SOURCE=1 -D__USE_LARGEFILE64) diff --git a/docker/packager/binary/Dockerfile b/docker/packager/binary/Dockerfile index b9b0c5c2c6c..c4244504923 100644 --- a/docker/packager/binary/Dockerfile +++ b/docker/packager/binary/Dockerfile @@ -67,24 +67,5 @@ ENV GOCACHE=/workdir/ RUN mkdir /workdir && chmod 777 /workdir WORKDIR /workdir -# NOTE: thread sanitizer is broken in clang-14, we have to build it with clang-15 -# https://github.com/ClickHouse/ClickHouse/pull/39450 -# https://github.com/google/sanitizers/issues/1540 -# https://github.com/google/sanitizers/issues/1552 - -RUN export CODENAME="$(lsb_release --codename --short | tr 'A-Z' 'a-z')" \ - && echo "deb [trusted=yes] https://apt.llvm.org/${CODENAME}/ llvm-toolchain-${CODENAME}-15 main" >> \ - /etc/apt/sources.list.d/clang.list \ - && apt-get update \ - && apt-get install \ - clang-15 \ - llvm-15 \ - clang-tidy-15 \ - --yes --no-install-recommends \ - && apt-get clean - -# for external_symbolizer_path -RUN ln -s /usr/bin/llvm-symbolizer-15 /usr/bin/llvm-symbolizer - COPY build.sh / CMD ["bash", "-c", "/build.sh 2>&1"] diff --git a/docker/packager/packager b/docker/packager/packager index 591262959b4..9da787e9006 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -339,17 +339,16 @@ if __name__ == "__main__": parser.add_argument( "--compiler", choices=( - "clang-15", # For TSAN builds, see #39450 - "clang-14", - "clang-14-darwin", - "clang-14-darwin-aarch64", - "clang-14-aarch64", - "clang-14-ppc64le", - "clang-14-amd64sse2", - "clang-14-freebsd", + "clang-15", + "clang-15-darwin", + "clang-15-darwin-aarch64", + "clang-15-aarch64", + "clang-15-ppc64le", + "clang-15-amd64sse2", + "clang-15-freebsd", "gcc-11", ), - default="clang-14", + default="clang-15", help="a compiler to use", ) parser.add_argument( diff --git a/docker/server/Dockerfile.alpine b/docker/server/Dockerfile.alpine index b01dba1e22f..1a672f30a74 100644 --- a/docker/server/Dockerfile.alpine +++ b/docker/server/Dockerfile.alpine @@ -33,7 +33,7 @@ RUN arch=${TARGETARCH:-amd64} \ # lts / testing / prestable / etc ARG REPO_CHANNEL="stable" ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}" -ARG VERSION="20.9.3.45" +ARG VERSION="22.8.5.29" ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static" # user/group precreated explicitly with fixed uid/gid on purpose. diff --git a/docker/server/Dockerfile.ubuntu b/docker/server/Dockerfile.ubuntu index f4102a6ccaf..db76a9fab1d 100644 --- a/docker/server/Dockerfile.ubuntu +++ b/docker/server/Dockerfile.ubuntu @@ -21,7 +21,7 @@ RUN sed -i "s|http://archive.ubuntu.com|${apt_archive}|g" /etc/apt/sources.list ARG REPO_CHANNEL="stable" ARG REPOSITORY="deb https://packages.clickhouse.com/deb ${REPO_CHANNEL} main" -ARG VERSION=22.6.1.* +ARG VERSION="22.8.5.29" ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static" # set non-empty deb_location_url url to create a docker image diff --git a/docker/test/base/Dockerfile b/docker/test/base/Dockerfile index 43cfca1fdfc..4e42fce1a1d 100644 --- a/docker/test/base/Dockerfile +++ b/docker/test/base/Dockerfile @@ -16,11 +16,10 @@ RUN apt-get update \ # and MEMORY_LIMIT_EXCEEDED exceptions in Functional tests (total memory limit in Functional tests is ~55.24 GiB). # TSAN will flush shadow memory when reaching this limit. # It may cause false-negatives, but it's better than OOM. -RUN echo "TSAN_OPTIONS='verbosity=1000 halt_on_error=1 history_size=7 memory_limit_mb=46080'" >> /etc/environment; \ - echo "UBSAN_OPTIONS='print_stacktrace=1'" >> /etc/environment; \ - echo "MSAN_OPTIONS='abort_on_error=1 poison_in_dtor=1'" >> /etc/environment; \ - echo "LSAN_OPTIONS='suppressions=/usr/share/clickhouse-test/config/lsan_suppressions.txt'" >> /etc/environment; \ - ln -s /usr/lib/llvm-${LLVM_VERSION}/bin/llvm-symbolizer /usr/bin/llvm-symbolizer; +RUN echo "TSAN_OPTIONS='verbosity=1000 halt_on_error=1 history_size=7 memory_limit_mb=46080'" >> /etc/environment +RUN echo "UBSAN_OPTIONS='print_stacktrace=1'" >> /etc/environment +RUN echo "MSAN_OPTIONS='abort_on_error=1 poison_in_dtor=1'" >> /etc/environment +RUN echo "LSAN_OPTIONS='suppressions=/usr/share/clickhouse-test/config/lsan_suppressions.txt'" >> /etc/environment # Sanitizer options for current shell (not current, but the one that will be spawned on "docker run") # (but w/o verbosity for TSAN, otherwise test.reference will not match) ENV TSAN_OPTIONS='halt_on_error=1 history_size=7 memory_limit_mb=46080' diff --git a/docker/test/codebrowser/Dockerfile b/docker/test/codebrowser/Dockerfile index c7aed618f6a..ceed93c3ac7 100644 --- a/docker/test/codebrowser/Dockerfile +++ b/docker/test/codebrowser/Dockerfile @@ -8,16 +8,41 @@ FROM clickhouse/binary-builder:$FROM_TAG ARG apt_archive="http://archive.ubuntu.com" RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list -RUN apt-get update && apt-get --yes --allow-unauthenticated install clang-14 libllvm14 libclang-14-dev libmlir-14-dev +RUN apt-get update && apt-get --yes --allow-unauthenticated install libclang-${LLVM_VERSION}-dev libmlir-${LLVM_VERSION}-dev + +# libclang-15-dev does not contain proper symlink: +# +# This is what cmake will search for: +# +# # readlink -f /usr/lib/llvm-15/lib/libclang-15.so.1 +# /usr/lib/x86_64-linux-gnu/libclang-15.so.1 +# +# This is what exists: +# +# # ls -l /usr/lib/x86_64-linux-gnu/libclang-15* +# lrwxrwxrwx 1 root root 16 Sep 5 13:31 /usr/lib/x86_64-linux-gnu/libclang-15.so -> libclang-15.so.1 +# lrwxrwxrwx 1 root root 21 Sep 5 13:31 /usr/lib/x86_64-linux-gnu/libclang-15.so.15 -> libclang-15.so.15.0.0 +# -rw-r--r-- 1 root root 31835760 Sep 5 13:31 /usr/lib/x86_64-linux-gnu/libclang-15.so.15.0.0 +# +ARG TARGETARCH +RUN arch=${TARGETARCH:-amd64} \ + && case $arch in \ + amd64) rarch=x86_64 ;; \ + arm64) rarch=aarch64 ;; \ + *) exit 1 ;; \ + esac \ + && ln -rsf /usr/lib/$rarch-linux-gnu/libclang-15.so.15 /usr/lib/$rarch-linux-gnu/libclang-15.so.1 # repo versions doesn't work correctly with C++17 # also we push reports to s3, so we add index.html to subfolder urls # https://github.com/ClickHouse-Extras/woboq_codebrowser/commit/37e15eaf377b920acb0b48dbe82471be9203f76b # TODO: remove branch in a few weeks after merge, e.g. in May or June 2022 -RUN git clone https://github.com/ClickHouse-Extras/woboq_codebrowser --branch llvm-14 \ +# +# FIXME: update location of a repo +RUN git clone https://github.com/azat/woboq_codebrowser --branch llvm-15 \ && cd woboq_codebrowser \ - && cmake . -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=clang\+\+-14 -DCMAKE_C_COMPILER=clang-14 \ - && make -j \ + && cmake . -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=clang\+\+-${LLVM_VERSION} -DCMAKE_C_COMPILER=clang-${LLVM_VERSION} \ + && ninja \ && cd .. \ && rm -rf woboq_codebrowser @@ -32,7 +57,7 @@ ENV SHA=nosha ENV DATA="https://s3.amazonaws.com/clickhouse-test-reports/codebrowser/data" CMD mkdir -p $BUILD_DIRECTORY && cd $BUILD_DIRECTORY && \ - cmake $SOURCE_DIRECTORY -DCMAKE_CXX_COMPILER=/usr/bin/clang\+\+-14 -DCMAKE_C_COMPILER=/usr/bin/clang-14 -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DENABLE_EMBEDDED_COMPILER=0 -DENABLE_S3=0 && \ + cmake $SOURCE_DIRECTORY -DCMAKE_CXX_COMPILER=/usr/bin/clang\+\+-${LLVM_VERSION} -DCMAKE_C_COMPILER=/usr/bin/clang-${LLVM_VERSION} -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DENABLE_EMBEDDED_COMPILER=0 -DENABLE_S3=0 && \ mkdir -p $HTML_RESULT_DIRECTORY && \ $CODEGEN -b $BUILD_DIRECTORY -a -o $HTML_RESULT_DIRECTORY -p ClickHouse:$SOURCE_DIRECTORY:$SHA -d $DATA | ts '%Y-%m-%d %H:%M:%S' && \ cp -r $STATIC_DATA $HTML_RESULT_DIRECTORY/ &&\ diff --git a/docker/test/fuzzer/run-fuzzer.sh b/docker/test/fuzzer/run-fuzzer.sh index 93e38260395..bab87865b42 100755 --- a/docker/test/fuzzer/run-fuzzer.sh +++ b/docker/test/fuzzer/run-fuzzer.sh @@ -19,7 +19,7 @@ stage=${stage:-} script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" echo "$script_dir" repo_dir=ch -BINARY_TO_DOWNLOAD=${BINARY_TO_DOWNLOAD:="clang-14_debug_none_unsplitted_disable_False_binary"} +BINARY_TO_DOWNLOAD=${BINARY_TO_DOWNLOAD:="clang-15_debug_none_unsplitted_disable_False_binary"} BINARY_URL_TO_DOWNLOAD=${BINARY_URL_TO_DOWNLOAD:="https://clickhouse-builds.s3.amazonaws.com/$PR_TO_TEST/$SHA_TO_TEST/clickhouse_build_check/$BINARY_TO_DOWNLOAD/clickhouse"} function clone diff --git a/docker/test/integration/runner/dockerd-entrypoint.sh b/docker/test/integration/runner/dockerd-entrypoint.sh index bcaa064fe4f..5ae880ddf36 100755 --- a/docker/test/integration/runner/dockerd-entrypoint.sh +++ b/docker/test/integration/runner/dockerd-entrypoint.sh @@ -28,10 +28,9 @@ done set -e # cleanup for retry run if volume is not recreated -# shellcheck disable=SC2046 { - docker ps -aq | xargs -r docker kill || true - docker ps -aq | xargs -r docker rm || true + docker ps --all --quiet | xargs --no-run-if-empty docker kill || true + docker ps --all --quiet | xargs --no-run-if-empty docker rm || true } echo "Start tests" diff --git a/docker/test/keeper-jepsen/run.sh b/docker/test/keeper-jepsen/run.sh index c43e6b2c54d..adf99c029a9 100644 --- a/docker/test/keeper-jepsen/run.sh +++ b/docker/test/keeper-jepsen/run.sh @@ -2,7 +2,7 @@ set -euo pipefail -CLICKHOUSE_PACKAGE=${CLICKHOUSE_PACKAGE:="https://clickhouse-builds.s3.amazonaws.com/$PR_TO_TEST/$SHA_TO_TEST/clickhouse_build_check/clang-14_relwithdebuginfo_none_unsplitted_disable_False_binary/clickhouse"} +CLICKHOUSE_PACKAGE=${CLICKHOUSE_PACKAGE:="https://clickhouse-builds.s3.amazonaws.com/$PR_TO_TEST/$SHA_TO_TEST/clickhouse_build_check/clang-15_relwithdebuginfo_none_unsplitted_disable_False_binary/clickhouse"} CLICKHOUSE_REPO_PATH=${CLICKHOUSE_REPO_PATH:=""} diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh index d3d7084f37f..b0b5ebdb2e2 100755 --- a/docker/test/performance-comparison/compare.sh +++ b/docker/test/performance-comparison/compare.sh @@ -61,7 +61,7 @@ function configure cp -rv right/config left ||: # Start a temporary server to rename the tables - while pkill clickhouse-serv; do echo . ; sleep 1 ; done + while pkill -f clickhouse-serv ; do echo . ; sleep 1 ; done echo all killed set -m # Spawn temporary in its own process groups @@ -88,7 +88,7 @@ function configure clickhouse-client --port $LEFT_SERVER_PORT --query "create database test" ||: clickhouse-client --port $LEFT_SERVER_PORT --query "rename table datasets.hits_v1 to test.hits" ||: - while pkill clickhouse-serv; do echo . ; sleep 1 ; done + while pkill -f clickhouse-serv ; do echo . ; sleep 1 ; done echo all killed # Make copies of the original db for both servers. Use hardlinks instead @@ -106,7 +106,7 @@ function configure function restart { - while pkill clickhouse-serv; do echo . ; sleep 1 ; done + while pkill -f clickhouse-serv ; do echo . ; sleep 1 ; done echo all killed # Change the jemalloc settings here. @@ -1400,7 +1400,7 @@ case "$stage" in while env kill -- -$watchdog_pid ; do sleep 1; done # Stop the servers to free memory for the subsequent query analysis. - while pkill clickhouse-serv; do echo . ; sleep 1 ; done + while pkill -f clickhouse-serv ; do echo . ; sleep 1 ; done echo Servers stopped. ;& "analyze_queries") diff --git a/docker/test/util/Dockerfile b/docker/test/util/Dockerfile index b891b71492c..57880bfc1d6 100644 --- a/docker/test/util/Dockerfile +++ b/docker/test/util/Dockerfile @@ -5,7 +5,7 @@ FROM ubuntu:20.04 ARG apt_archive="http://archive.ubuntu.com" RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list -ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=14 +ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=15 RUN apt-get update \ && apt-get install \ @@ -56,6 +56,8 @@ RUN apt-get update \ # This symlink required by gcc to find lld compiler RUN ln -s /usr/bin/lld-${LLVM_VERSION} /usr/bin/ld.lld +# for external_symbolizer_path +RUN ln -s /usr/bin/llvm-symbolizer-${LLVM_VERSION} /usr/bin/llvm-symbolizer ARG CCACHE_VERSION=4.6.1 RUN mkdir /tmp/ccache \ diff --git a/docs/changelogs/v22.8.5.29-lts.md b/docs/changelogs/v22.8.5.29-lts.md new file mode 100644 index 00000000000..0ce13b7c36e --- /dev/null +++ b/docs/changelogs/v22.8.5.29-lts.md @@ -0,0 +1,34 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + +### ClickHouse release v22.8.5.29-lts (74ffb843807) FIXME as compared to v22.8.4.7-lts (baad27bcd2f) + +#### New Feature +* Backported in [#40870](https://github.com/ClickHouse/ClickHouse/issues/40870): Add setting to disable limit on kafka_num_consumers. Closes [#40331](https://github.com/ClickHouse/ClickHouse/issues/40331). [#40670](https://github.com/ClickHouse/ClickHouse/pull/40670) ([Kruglov Pavel](https://github.com/Avogar)). + +#### Improvement +* Backported in [#40817](https://github.com/ClickHouse/ClickHouse/issues/40817): The setting `show_addresses_in_stack_traces` was accidentally disabled in default `config.xml`. It's removed from the config now, so the setting is enabled by default. [#40749](https://github.com/ClickHouse/ClickHouse/pull/40749) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#40944](https://github.com/ClickHouse/ClickHouse/issues/40944): Fix issue with passing MySQL timeouts for MySQL database engine and MySQL table function. Closes [#34168](https://github.com/ClickHouse/ClickHouse/issues/34168)?notification_referrer_id=NT_kwDOAzsV57MzMDMxNjAzNTY5OjU0MjAzODc5. [#40751](https://github.com/ClickHouse/ClickHouse/pull/40751) ([Kseniia Sumarokova](https://github.com/kssenii)). + +#### Build/Testing/Packaging Improvement +* Backported in [#41157](https://github.com/ClickHouse/ClickHouse/issues/41157): Add macOS binaries to GH release assets, it fixes [#37718](https://github.com/ClickHouse/ClickHouse/issues/37718). [#41088](https://github.com/ClickHouse/ClickHouse/pull/41088) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). + +#### Bug Fix (user-visible misbehavior in official stable or prestable release) + +* Backported in [#40866](https://github.com/ClickHouse/ClickHouse/issues/40866): - Fix crash while parsing values of type `Object` that contains arrays of variadic dimension. [#40483](https://github.com/ClickHouse/ClickHouse/pull/40483) ([Duc Canh Le](https://github.com/canhld94)). +* Backported in [#40805](https://github.com/ClickHouse/ClickHouse/issues/40805): During insertion of a new query to the `ProcessList` allocations happen. If we reach the memory limit during these allocations we can not use `OvercommitTracker`, because `ProcessList::mutex` is already acquired. Fixes [#40611](https://github.com/ClickHouse/ClickHouse/issues/40611). [#40677](https://github.com/ClickHouse/ClickHouse/pull/40677) ([Dmitry Novik](https://github.com/novikd)). +* Backported in [#40777](https://github.com/ClickHouse/ClickHouse/issues/40777): Fix memory leak while pushing to MVs w/o query context (from Kafka/...). [#40732](https://github.com/ClickHouse/ClickHouse/pull/40732) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#41135](https://github.com/ClickHouse/ClickHouse/issues/41135): Fix access rights for `DESCRIBE TABLE url()` and some other `DESCRIBE TABLE ()`. [#40975](https://github.com/ClickHouse/ClickHouse/pull/40975) ([Vitaly Baranov](https://github.com/vitlibar)). +* Backported in [#41242](https://github.com/ClickHouse/ClickHouse/issues/41242): Fixed "possible deadlock avoided" error on automatic conversion of database engine from Ordinary to Atomic. [#41146](https://github.com/ClickHouse/ClickHouse/pull/41146) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#41234](https://github.com/ClickHouse/ClickHouse/issues/41234): Fix background clean up of broken detached parts. [#41190](https://github.com/ClickHouse/ClickHouse/pull/41190) ([Kseniia Sumarokova](https://github.com/kssenii)). + +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* use ROBOT_CLICKHOUSE_COMMIT_TOKEN for create-pull-request [#40067](https://github.com/ClickHouse/ClickHouse/pull/40067) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). +* use input token instead of env var [#40421](https://github.com/ClickHouse/ClickHouse/pull/40421) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). +* CaresPTRResolver small safety improvement [#40890](https://github.com/ClickHouse/ClickHouse/pull/40890) ([Arthur Passos](https://github.com/arthurpassos)). + diff --git a/docs/en/engines/table-engines/special/materializedview.md b/docs/en/engines/table-engines/special/materializedview.md index 7b06560ec98..807f7bc9eae 100644 --- a/docs/en/engines/table-engines/special/materializedview.md +++ b/docs/en/engines/table-engines/special/materializedview.md @@ -8,4 +8,4 @@ sidebar_label: MaterializedView Used for implementing materialized views (for more information, see [CREATE VIEW](../../../sql-reference/statements/create/view.md#materialized)). For storing data, it uses a different engine that was specified when creating the view. When reading from a table, it just uses that engine. -[Original article](https://clickhouse.com/docs/en/engines/table-engines/special/materializedview/) +[Original article](https://clickhouse.com/docs/en/sql-reference/statements/create/view#materialized-view) diff --git a/docs/en/interfaces/formats.md b/docs/en/interfaces/formats.md index 640c49377d0..36e30360384 100644 --- a/docs/en/interfaces/formats.md +++ b/docs/en/interfaces/formats.md @@ -2,10 +2,9 @@ slug: /en/interfaces/formats sidebar_position: 21 sidebar_label: Input and Output Formats +title: Formats for Input and Output Data --- -# Formats for Input and Output Data - ClickHouse can accept and return data in various formats. A format supported for input can be used to parse the data provided to `INSERT`s, to perform `SELECT`s from a file-backed table such as File, URL or HDFS, or to read an external dictionary. A format supported for output can be used to arrange the results of a `SELECT`, and to perform `INSERT`s into a file-backed table. diff --git a/docs/en/interfaces/third-party/integrations.md b/docs/en/interfaces/third-party/integrations.md index de496546cb4..aede128e9a4 100644 --- a/docs/en/interfaces/third-party/integrations.md +++ b/docs/en/interfaces/third-party/integrations.md @@ -103,6 +103,7 @@ ClickHouse, Inc. does **not** maintain the tools and libraries listed below and - [ClickHouse.Client](https://github.com/DarkWanderer/ClickHouse.Client) - [ClickHouse.Net](https://github.com/ilyabreev/ClickHouse.Net) - [ClickHouse.Net.Migrations](https://github.com/ilyabreev/ClickHouse.Net.Migrations) + - [Linq To DB](https://github.com/linq2db/linq2db) - Elixir - [Ecto](https://github.com/elixir-ecto/ecto) - [clickhouse_ecto](https://github.com/appodeal/clickhouse_ecto) diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index 3869168becd..dde40acb91a 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -3145,6 +3145,17 @@ Result: └─────┴─────┴───────┘ ``` +## enable_extended_results_for_datetime_functions {#enable-extended-results-for-datetime-functions} + +Enables or disables returning results of type `Date32` with extended range (compared to type `Date`) for functions [toStartOfYear](../../sql-reference/functions/date-time-functions.md#tostartofyear), [toStartOfISOYear](../../sql-reference/functions/date-time-functions.md#tostartofisoyear), [toStartOfQuarter](../../sql-reference/functions/date-time-functions.md#tostartofquarter), [toStartOfMonth](../../sql-reference/functions/date-time-functions.md#tostartofmonth), [toStartOfWeek](../../sql-reference/functions/date-time-functions.md#tostartofweek), [toMonday](../../sql-reference/functions/date-time-functions.md#tomonday) and [toLastDayOfMonth](../../sql-reference/functions/date-time-functions.md#tolastdayofmonth). + +Possible values: + +- 0 — Functions return `Date` for all types of arguments. +- 1 — Functions return `Date32` for `Date32` or `DateTime64` arguments and `Date` otherwise. + +Default value: `0`. + ## optimize_move_to_prewhere {#optimize_move_to_prewhere} Enables or disables automatic [PREWHERE](../../sql-reference/statements/select/prewhere.md) optimization in [SELECT](../../sql-reference/statements/select/index.md) queries. @@ -3530,8 +3541,8 @@ desc format(JSONEachRow, '{"x" : 1, "y" : "String", "z" : "0.0.0.0" }') settings Result: ```sql -x UInt8 -y Nullable(String) +x UInt8 +y Nullable(String) z IPv4 ``` diff --git a/docs/en/operations/storing-data.md b/docs/en/operations/storing-data.md index 546e3d7b7a6..663469ef4ae 100644 --- a/docs/en/operations/storing-data.md +++ b/docs/en/operations/storing-data.md @@ -134,6 +134,13 @@ Example of configuration for versions later or equal to 22.8: 10000000 + + +
+ cache +
+ + ``` @@ -151,6 +158,13 @@ Example of configuration for versions earlier than 22.8: 10000000 + + +
+ s3 +
+ + ``` @@ -166,7 +180,7 @@ Cache **configuration settings**: - `enable_cache_hits_threshold` - a number, which defines how many times some data needs to be read before it will be cached. Default: `0`, e.g. the data is cached at the first attempt to read it. -- `do_not_evict_index_and_mark_files` - do not evict small frequently used files according to cache policy. Default: `true`. +- `do_not_evict_index_and_mark_files` - do not evict small frequently used files according to cache policy. Default: `false`. This setting was added in version 22.8. If you used filesystem cache before this version, then it will not work on versions starting from 22.8 if this setting is set to `true`. If you want to use this setting, clear old cache created before version 22.8 before upgrading. - `max_file_segment_size` - a maximum size of a single cache file. Default: `104857600` (100 Mb). diff --git a/docs/en/operations/troubleshooting.md b/docs/en/operations/troubleshooting.md index 5a61359a2c0..93bd56087a2 100644 --- a/docs/en/operations/troubleshooting.md +++ b/docs/en/operations/troubleshooting.md @@ -2,10 +2,9 @@ slug: /en/operations/troubleshooting sidebar_position: 46 sidebar_label: Troubleshooting +title: Troubleshooting --- -# Troubleshooting - - [Installation](#troubleshooting-installation-errors) - [Connecting to the server](#troubleshooting-accepts-no-connections) - [Query processing](#troubleshooting-does-not-process-queries) diff --git a/docs/en/sql-reference/data-types/geo.md b/docs/en/sql-reference/data-types/geo.md index 65fd641bec9..48dce40986e 100644 --- a/docs/en/sql-reference/data-types/geo.md +++ b/docs/en/sql-reference/data-types/geo.md @@ -7,13 +7,8 @@ title: "Geo Data Types" ClickHouse supports data types for representing geographical objects — locations, lands, etc. -:::warning -Currently geo data types are an experimental feature. To work with them you must set `allow_experimental_geo_types = 1`. -::: - **See Also** - [Representing simple geographical features](https://en.wikipedia.org/wiki/GeoJSON). -- [allow_experimental_geo_types](../../operations/settings/settings.md#allow-experimental-geo-types) setting. ## Point @@ -24,7 +19,6 @@ Currently geo data types are an experimental feature. To work with them you must Query: ```sql -SET allow_experimental_geo_types = 1; CREATE TABLE geo_point (p Point) ENGINE = Memory(); INSERT INTO geo_point VALUES((10, 10)); SELECT p, toTypeName(p) FROM geo_point; @@ -46,7 +40,6 @@ Result: Query: ```sql -SET allow_experimental_geo_types = 1; CREATE TABLE geo_ring (r Ring) ENGINE = Memory(); INSERT INTO geo_ring VALUES([(0, 0), (10, 0), (10, 10), (0, 10)]); SELECT r, toTypeName(r) FROM geo_ring; @@ -68,7 +61,6 @@ Result: This is a polygon with one hole: ```sql -SET allow_experimental_geo_types = 1; CREATE TABLE geo_polygon (pg Polygon) ENGINE = Memory(); INSERT INTO geo_polygon VALUES([[(20, 20), (50, 20), (50, 50), (20, 50)], [(30, 30), (50, 50), (50, 30)]]); SELECT pg, toTypeName(pg) FROM geo_polygon; @@ -91,7 +83,6 @@ Result: This multipolygon consists of two separate polygons — the first one without holes, and the second with one hole: ```sql -SET allow_experimental_geo_types = 1; CREATE TABLE geo_multipolygon (mpg MultiPolygon) ENGINE = Memory(); INSERT INTO geo_multipolygon VALUES([[[(0, 0), (10, 0), (10, 10), (0, 10)]], [[(20, 20), (50, 20), (50, 50), (20, 50)],[(30, 30), (50, 50), (50, 30)]]]); SELECT mpg, toTypeName(mpg) FROM geo_multipolygon; diff --git a/docs/en/sql-reference/formats.mdx b/docs/en/sql-reference/formats.mdx new file mode 100644 index 00000000000..013b182091b --- /dev/null +++ b/docs/en/sql-reference/formats.mdx @@ -0,0 +1,10 @@ +--- +slug: /en/sql-reference/formats +sidebar_position: 50 +sidebar_label: Input and Output Formats +title: Formats for Input and Output Data +--- + +import Content from '@site/docs/en/interfaces/formats.md'; + + diff --git a/docs/en/sql-reference/functions/date-time-functions.md b/docs/en/sql-reference/functions/date-time-functions.md index ced96078ce1..001c7822433 100644 --- a/docs/en/sql-reference/functions/date-time-functions.md +++ b/docs/en/sql-reference/functions/date-time-functions.md @@ -268,13 +268,15 @@ Result: ``` :::note -The return type of `toStartOf*`, `toLastDayOfMonth`, `toMonday` functions described below is `Date` or `DateTime`. -Though these functions can take values of the extended types `Date32` and `DateTime64` as an argument, passing them a time outside the normal range (year 1970 to 2149 for `Date` / 2106 for `DateTime`) will produce wrong results. -In case argument is out of normal range: +The return type of `toStartOf*`, `toLastDayOfMonth`, `toMonday` functions described below is determined by the configuration parameter [enable_extended_results_for_datetime_functions](../../operations/settings/settings#enable-extended-results-for-datetime-functions) which is `0` by default. + +Behavior for +* `enable_extended_results_for_datetime_functions = 0`: Functions `toStartOf*`, `toLastDayOfMonth`, `toMonday` return `Date` or `DateTime`. Though these functions can take values of the extended types `Date32` and `DateTime64` as an argument, passing them a time outside the normal range (year 1970 to 2149 for `Date` / 2106 for `DateTime`) will produce wrong results. In case argument is out of normal range: * If the argument is smaller than 1970, the result will be calculated from the argument `1970-01-01 (00:00:00)` instead. * If the return type is `DateTime` and the argument is larger than `2106-02-07 08:28:15`, the result will be calculated from the argument `2106-02-07 08:28:15` instead. * If the return type is `Date` and the argument is larger than `2149-06-06`, the result will be calculated from the argument `2149-06-06` instead. * If `toLastDayOfMonth` is called with an argument greater then `2149-05-31`, the result will be calculated from the argument `2149-05-31` instead. +* `enable_extended_results_for_datetime_functions = 1`: Functions `toStartOfYear`, `toStartOfISOYear`, `toStartOfQuarter`, `toStartOfMonth`, `toStartOfWeek`, `toLastDayOfMonth`, `toMonday` return `Date` or `DateTime` if their argument is a `Date` or `DateTime`, and they return `Date32` or `DateTime64` if their argument is a `Date32` or `DateTime64`. ::: ## toStartOfYear @@ -303,6 +305,8 @@ Returns the date. Rounds up a date or date with time to the last day of the month. Returns the date. +If `toLastDayOfMonth` is called with an argument of type `Date` greater then 2149-05-31, the result will be calculated from the argument 2149-05-31 instead. + ## toMonday Rounds down a date or date with time to the nearest Monday. @@ -640,7 +644,7 @@ Result: ## date\_diff -Returns the difference between two dates or dates with time values. +Returns the difference between two dates or dates with time values. The difference is calculated using relative units, e.g. the difference between `2022-01-01` and `2021-12-29` is 3 days for day unit (see [toRelativeDayNum](#torelativedaynum)), 1 month for month unit (see [toRelativeMonthNum](#torelativemonthnum)), 1 year for year unit (see [toRelativeYearNum](#torelativeyearnum)). **Syntax** @@ -1059,9 +1063,9 @@ SELECT ## timeSlots(StartTime, Duration,\[, Size\]) -For a time interval starting at ‘StartTime’ and continuing for ‘Duration’ seconds, it returns an array of moments in time, consisting of points from this interval rounded down to the ‘Size’ in seconds. ‘Size’ is an optional parameter set to 1800 (30 minutes) by default. -This is necessary, for example, when searching for pageviews in the corresponding session. -Accepts DateTime and DateTime64 as ’StartTime’ argument. For DateTime, ’Duration’ and ’Size’ arguments must be `UInt32`. For ’DateTime64’ they must be `Decimal64`. +For a time interval starting at ‘StartTime’ and continuing for ‘Duration’ seconds, it returns an array of moments in time, consisting of points from this interval rounded down to the ‘Size’ in seconds. ‘Size’ is an optional parameter set to 1800 (30 minutes) by default. +This is necessary, for example, when searching for pageviews in the corresponding session. +Accepts DateTime and DateTime64 as ’StartTime’ argument. For DateTime, ’Duration’ and ’Size’ arguments must be `UInt32`. For ’DateTime64’ they must be `Decimal64`. Returns an array of DateTime/DateTime64 (return type matches the type of ’StartTime’). For DateTime64, the return value's scale can differ from the scale of ’StartTime’ --- the highest scale among all given arguments is taken. Example: @@ -1227,6 +1231,8 @@ Result: Function converts Unix timestamp to a calendar date and a time of a day. When there is only a single argument of [Integer](../../sql-reference/data-types/int-uint.md) type, it acts in the same way as [toDateTime](../../sql-reference/functions/type-conversion-functions.md#todatetime) and return [DateTime](../../sql-reference/data-types/datetime.md) type. +Alias: `fromUnixTimestamp`. + **Example:** Query: diff --git a/docs/en/sql-reference/functions/other-functions.md b/docs/en/sql-reference/functions/other-functions.md index d86eb6b45ae..877179a66a6 100644 --- a/docs/en/sql-reference/functions/other-functions.md +++ b/docs/en/sql-reference/functions/other-functions.md @@ -1823,6 +1823,36 @@ Result: Evaluate external model. Accepts a model name and model arguments. Returns Float64. +## catboostEvaluate(path_to_model, feature_1, feature_2, …, feature_n) + +Evaluate external catboost model. [CatBoost](https://catboost.ai) is an open-source gradient boosting library developed by Yandex for machine learing. +Accepts a path to a catboost model and model arguments (features). Returns Float64. + +``` sql +SELECT feat1, ..., feat_n, catboostEvaluate('/path/to/model.bin', feat_1, ..., feat_n) AS prediction +FROM data_table +``` + +**Prerequisites** + +1. Build the catboost evaluation library + +Before evaluating catboost models, the `libcatboostmodel.` library must be made available. See [CatBoost documentation](https://catboost.ai/docs/concepts/c-plus-plus-api_dynamic-c-pluplus-wrapper.html) how to compile it. + +Next, specify the path to `libcatboostmodel.` in the clickhouse configuration: + +``` xml + +... + /path/to/libcatboostmodel.so +... + +``` + +2. Train a catboost model using libcatboost + +See [Training and applying models](https://catboost.ai/docs/features/training.html#training) for how to train catboost models from a training data set. + ## throwIf(x\[, message\[, error_code\]\]) Throw an exception if the argument is non zero. diff --git a/docs/en/sql-reference/statements/delete.md b/docs/en/sql-reference/statements/delete.md index 487dfc87f9a..0dc6cc0d09a 100644 --- a/docs/en/sql-reference/statements/delete.md +++ b/docs/en/sql-reference/statements/delete.md @@ -32,6 +32,12 @@ SET allow_experimental_lightweight_delete = true; An [alternative way to delete rows](./alter/delete.md) in ClickHouse is `ALTER TABLE ... DELETE`, which might be more efficient if you do bulk deletes only occasionally and don't need the operation to be applied instantly. In most use cases the new lightweight `DELETE FROM` behavior will be considerably faster. :::warning -Even though deletes are becoming more lightweight in ClickHouse, they should still not be used as aggressively as on OLTP system. Ligthweight deletes are currently efficient for wide parts, but for compact parts they can be a heavyweight operation, and it may be better to use `ALTER TABLE` for some scenarios. +Even though deletes are becoming more lightweight in ClickHouse, they should still not be used as aggressively as on an OLTP system. Ligthweight deletes are currently efficient for wide parts, but for compact parts they can be a heavyweight operation, and it may be better to use `ALTER TABLE` for some scenarios. ::: +:::note +`DELETE FROM` requires the `ALTER DELETE` privilege: +```sql +grant ALTER DELETE ON db.table to username; +``` +::: diff --git a/docs/en/sql-reference/statements/select/group-by.md b/docs/en/sql-reference/statements/select/group-by.md index b5e194343ca..ac02e9ab5a1 100644 --- a/docs/en/sql-reference/statements/select/group-by.md +++ b/docs/en/sql-reference/statements/select/group-by.md @@ -213,9 +213,10 @@ If the `WITH TOTALS` modifier is specified, another row will be calculated. This This extra row is only produced in `JSON*`, `TabSeparated*`, and `Pretty*` formats, separately from the other rows: -- In `JSON*` formats, this row is output as a separate ‘totals’ field. -- In `TabSeparated*` formats, the row comes after the main result, preceded by an empty row (after the other data). +- In `XML` and `JSON*` formats, this row is output as a separate ‘totals’ field. +- In `TabSeparated*`, `CSV*` and `Vertical` formats, the row comes after the main result, preceded by an empty row (after the other data). - In `Pretty*` formats, the row is output as a separate table after the main result. +- In `Template` format, the row is output according to specified template. - In the other formats it is not available. :::note diff --git a/docs/en/sql-reference/statements/select/index.md b/docs/en/sql-reference/statements/select/index.md index c1692bd9b29..de190aaf982 100644 --- a/docs/en/sql-reference/statements/select/index.md +++ b/docs/en/sql-reference/statements/select/index.md @@ -135,9 +135,9 @@ In all other cases, we do not recommend using the asterisk, since it only gives In addition to results, you can also get minimum and maximum values for the results columns. To do this, set the **extremes** setting to 1. Minimums and maximums are calculated for numeric types, dates, and dates with times. For other columns, the default values are output. -An extra two rows are calculated – the minimums and maximums, respectively. These extra two rows are output in `JSON*`, `TabSeparated*`, and `Pretty*` [formats](../../../interfaces/formats.md), separate from the other rows. They are not output for other formats. +An extra two rows are calculated – the minimums and maximums, respectively. These extra two rows are output in `XML`, `JSON*`, `TabSeparated*`, `CSV*`, `Vertical`, `Template` and `Pretty*` [formats](../../../interfaces/formats.md), separate from the other rows. They are not output for other formats. -In `JSON*` formats, the extreme values are output in a separate ‘extremes’ field. In `TabSeparated*` formats, the row comes after the main result, and after ‘totals’ if present. It is preceded by an empty row (after the other data). In `Pretty*` formats, the row is output as a separate table after the main result, and after `totals` if present. +In `JSON*` and `XML` formats, the extreme values are output in a separate ‘extremes’ field. In `TabSeparated*`, `CSV*` and `Vertical` formats, the row comes after the main result, and after ‘totals’ if present. It is preceded by an empty row (after the other data). In `Pretty*` formats, the row is output as a separate table after the main result, and after `totals` if present. In `Template` format the extreme values are output according to specified template. Extreme values are calculated for rows before `LIMIT`, but after `LIMIT BY`. However, when using `LIMIT offset, size`, the rows before `offset` are included in `extremes`. In stream requests, the result may also include a small number of rows that passed through `LIMIT`. diff --git a/docs/en/sql-reference/statements/system.md b/docs/en/sql-reference/statements/system.md index 9b7527caaa9..feeefd5502a 100644 --- a/docs/en/sql-reference/statements/system.md +++ b/docs/en/sql-reference/statements/system.md @@ -6,45 +6,6 @@ sidebar_label: SYSTEM # SYSTEM Statements -The list of available `SYSTEM` statements: - -- [RELOAD EMBEDDED DICTIONARIES](#query_language-system-reload-emdedded-dictionaries) -- [RELOAD DICTIONARIES](#query_language-system-reload-dictionaries) -- [RELOAD DICTIONARY](#query_language-system-reload-dictionary) -- [RELOAD MODELS](#query_language-system-reload-models) -- [RELOAD MODEL](#query_language-system-reload-model) -- [RELOAD FUNCTIONS](#query_language-system-reload-functions) -- [RELOAD FUNCTION](#query_language-system-reload-functions) -- [DROP DNS CACHE](#query_language-system-drop-dns-cache) -- [DROP MARK CACHE](#query_language-system-drop-mark-cache) -- [DROP UNCOMPRESSED CACHE](#query_language-system-drop-uncompressed-cache) -- [DROP COMPILED EXPRESSION CACHE](#query_language-system-drop-compiled-expression-cache) -- [DROP REPLICA](#query_language-system-drop-replica) -- [FLUSH LOGS](#query_language-system-flush_logs) -- [RELOAD CONFIG](#query_language-system-reload-config) -- [SHUTDOWN](#query_language-system-shutdown) -- [KILL](#query_language-system-kill) -- [STOP DISTRIBUTED SENDS](#query_language-system-stop-distributed-sends) -- [FLUSH DISTRIBUTED](#query_language-system-flush-distributed) -- [START DISTRIBUTED SENDS](#query_language-system-start-distributed-sends) -- [STOP MERGES](#query_language-system-stop-merges) -- [START MERGES](#query_language-system-start-merges) -- [STOP TTL MERGES](#query_language-stop-ttl-merges) -- [START TTL MERGES](#query_language-start-ttl-merges) -- [STOP MOVES](#query_language-stop-moves) -- [START MOVES](#query_language-start-moves) -- [SYSTEM UNFREEZE](#query_language-system-unfreeze) -- [STOP FETCHES](#query_language-system-stop-fetches) -- [START FETCHES](#query_language-system-start-fetches) -- [STOP REPLICATED SENDS](#query_language-system-start-replicated-sends) -- [START REPLICATED SENDS](#query_language-system-start-replicated-sends) -- [STOP REPLICATION QUEUES](#query_language-system-stop-replication-queues) -- [START REPLICATION QUEUES](#query_language-system-start-replication-queues) -- [SYNC REPLICA](#query_language-system-sync-replica) -- [RESTART REPLICA](#query_language-system-restart-replica) -- [RESTORE REPLICA](#query_language-system-restore-replica) -- [RESTART REPLICAS](#query_language-system-restart-replicas) - ## RELOAD EMBEDDED DICTIONARIES Reload all [Internal dictionaries](../../sql-reference/dictionaries/internal-dicts.md). @@ -69,7 +30,12 @@ SELECT name, status FROM system.dictionaries; ## RELOAD MODELS -Reloads all [CatBoost](../../guides/developer/apply-catboost-model.md) models if the configuration was updated without restarting the server. +:::note +This statement and `SYSTEM RELOAD MODEL` merely unload catboost models from the clickhouse-library-bridge. The function `catboostEvaluate()` +loads a model upon first access if it is not loaded yet. +::: + +Unloads all CatBoost models. **Syntax** @@ -79,12 +45,12 @@ SYSTEM RELOAD MODELS [ON CLUSTER cluster_name] ## RELOAD MODEL -Completely reloads a CatBoost model `model_name` if the configuration was updated without restarting the server. +Unloads a CatBoost model at `model_path`. **Syntax** ```sql -SYSTEM RELOAD MODEL [ON CLUSTER cluster_name] +SYSTEM RELOAD MODEL [ON CLUSTER cluster_name] ``` ## RELOAD FUNCTIONS diff --git a/docs/en/sql-reference/table-functions/file.md b/docs/en/sql-reference/table-functions/file.md index a110bfbd15c..f40107aaaca 100644 --- a/docs/en/sql-reference/table-functions/file.md +++ b/docs/en/sql-reference/table-functions/file.md @@ -13,7 +13,7 @@ Creates a table from a file. This table function is similar to [url](../../sql-r **Syntax** ``` sql -file(path, format, structure) +file(path [,format] [,structure]) ``` **Parameters** diff --git a/docs/en/sql-reference/table-functions/s3.md b/docs/en/sql-reference/table-functions/s3.md index 2df7d6e46b3..545037665bb 100644 --- a/docs/en/sql-reference/table-functions/s3.md +++ b/docs/en/sql-reference/table-functions/s3.md @@ -11,7 +11,7 @@ Provides table-like interface to select/insert files in [Amazon S3](https://aws. **Syntax** ``` sql -s3(path, [aws_access_key_id, aws_secret_access_key,] format, structure, [compression]) +s3(path [,aws_access_key_id, aws_secret_access_key] [,format] [,structure] [,compression]) ``` **Arguments** diff --git a/docs/en/sql-reference/table-functions/s3Cluster.md b/docs/en/sql-reference/table-functions/s3Cluster.md index 9d006af9572..b81fc51fd18 100644 --- a/docs/en/sql-reference/table-functions/s3Cluster.md +++ b/docs/en/sql-reference/table-functions/s3Cluster.md @@ -10,7 +10,7 @@ Allows processing files from [Amazon S3](https://aws.amazon.com/s3/) in parallel **Syntax** ``` sql -s3Cluster(cluster_name, source, [access_key_id, secret_access_key,] format, structure) +s3Cluster(cluster_name, source, [,access_key_id, secret_access_key] [,format] [,structure]) ``` **Arguments** diff --git a/docs/en/sql-reference/table-functions/url.md b/docs/en/sql-reference/table-functions/url.md index f1ed7b4dfe4..014dc3ae853 100644 --- a/docs/en/sql-reference/table-functions/url.md +++ b/docs/en/sql-reference/table-functions/url.md @@ -13,7 +13,7 @@ sidebar_label: url **Syntax** ``` sql -url(URL, format, structure) +url(URL [,format] [,structure]) ``` **Parameters** diff --git a/docs/redirects.txt b/docs/redirects.txt index 949b9d48ca8..cea138f7237 100644 --- a/docs/redirects.txt +++ b/docs/redirects.txt @@ -155,7 +155,6 @@ getting_started/index.md getting-started/index.md getting_started/install.md getting-started/install.md getting_started/playground.md getting-started/playground.md getting_started/tutorial.md getting-started/tutorial.md -guides/apply_catboost_model.md guides/apply-catboost-model.md images/column_oriented.gif images/column-oriented.gif images/row_oriented.gif images/row-oriented.gif interfaces/http_interface.md interfaces/http.md diff --git a/docs/ru/guides/apply-catboost-model.md b/docs/ru/guides/apply-catboost-model.md deleted file mode 100644 index 68d7042df2d..00000000000 --- a/docs/ru/guides/apply-catboost-model.md +++ /dev/null @@ -1,241 +0,0 @@ ---- -slug: /ru/guides/apply-catboost-model -sidebar_position: 41 -sidebar_label: "Применение модели CatBoost в ClickHouse" ---- - -# Применение модели CatBoost в ClickHouse {#applying-catboost-model-in-clickhouse} - -[CatBoost](https://catboost.ai) — открытая программная библиотека разработанная компанией [Яндекс](https://yandex.ru/company/) для машинного обучения, которая использует схему градиентного бустинга. - -С помощью этой инструкции вы научитесь применять предобученные модели в ClickHouse: в результате вы запустите вывод модели из SQL. - -Чтобы применить модель CatBoost в ClickHouse: - -1. [Создайте таблицу](#create-table). -2. [Вставьте данные в таблицу](#insert-data-to-table). -3. [Интегрируйте CatBoost в ClickHouse](#integrate-catboost-into-clickhouse) (Опциональный шаг). -4. [Запустите вывод модели из SQL](#run-model-inference). - -Подробнее об обучении моделей в CatBoost, см. [Обучение и применение моделей](https://catboost.ai/docs/features/training.html#training). - -Вы можете перегрузить модели CatBoost, если их конфигурация была обновлена, без перезагрузки сервера. Для этого используйте системные запросы [RELOAD MODEL](../sql-reference/statements/system.md#query_language-system-reload-model) и [RELOAD MODELS](../sql-reference/statements/system.md#query_language-system-reload-models). - -## Перед началом работы {#prerequisites} - -Если у вас еще нет [Docker](https://docs.docker.com/install/), установите его. - - :::note "Примечание" - [Docker](https://www.docker.com) – это программная платформа для создания контейнеров, которые изолируют установку CatBoost и ClickHouse от остальной части системы. - ::: -Перед применением модели CatBoost: - -**1.** Скачайте [Docker-образ](https://hub.docker.com/r/yandex/tutorial-catboost-clickhouse) из реестра: - -``` bash -$ docker pull yandex/tutorial-catboost-clickhouse -``` - -Данный Docker-образ содержит все необходимое для запуска CatBoost и ClickHouse: код, среду выполнения, библиотеки, переменные окружения и файлы конфигурации. - -**2.** Проверьте, что Docker-образ успешно скачался: - -``` bash -$ docker image ls -REPOSITORY TAG IMAGE ID CREATED SIZE -yandex/tutorial-catboost-clickhouse latest 622e4d17945b 22 hours ago 1.37GB -``` - -**3.** Запустите Docker-контейнер основанный на данном образе: - -``` bash -$ docker run -it -p 8888:8888 yandex/tutorial-catboost-clickhouse -``` - -## 1. Создайте таблицу {#create-table} - -Чтобы создать таблицу для обучающей выборки: - -**1.** Запустите клиент ClickHouse: - -``` bash -$ clickhouse client -``` - - :::note "Примечание" - Сервер ClickHouse уже запущен внутри Docker-контейнера. - ::: -**2.** Создайте таблицу в ClickHouse с помощью следующей команды: - -``` sql -:) CREATE TABLE amazon_train -( - date Date MATERIALIZED today(), - ACTION UInt8, - RESOURCE UInt32, - MGR_ID UInt32, - ROLE_ROLLUP_1 UInt32, - ROLE_ROLLUP_2 UInt32, - ROLE_DEPTNAME UInt32, - ROLE_TITLE UInt32, - ROLE_FAMILY_DESC UInt32, - ROLE_FAMILY UInt32, - ROLE_CODE UInt32 -) -ENGINE = MergeTree ORDER BY date -``` - -**3.** Выйдите из клиента ClickHouse: - -``` sql -:) exit -``` - -## 2. Вставьте данные в таблицу {#insert-data-to-table} - -Чтобы вставить данные: - -**1.** Выполните следующую команду: - -``` bash -$ clickhouse client --host 127.0.0.1 --query 'INSERT INTO amazon_train FORMAT CSVWithNames' < ~/amazon/train.csv -``` - -**2.** Запустите клиент ClickHouse: - -``` bash -$ clickhouse client -``` - -**3.** Проверьте, что данные успешно загрузились: - -``` sql -:) SELECT count() FROM amazon_train - -SELECT count() -FROM amazon_train - -+-count()-+ -| 65538 | -+---------+ -``` - -## 3. Интегрируйте CatBoost в ClickHouse {#integrate-catboost-into-clickhouse} - - :::note "Примечание" - **Опциональный шаг.** Docker-образ содержит все необходимое для запуска CatBoost и ClickHouse. - ::: -Чтобы интегрировать CatBoost в ClickHouse: - -**1.** Создайте библиотеку для оценки модели. - -Наиболее быстрый способ оценить модель CatBoost — это скомпилировать библиотеку `libcatboostmodel.`. Подробнее о том, как скомпилировать библиотеку, читайте в [документации CatBoost](https://catboost.ai/docs/concepts/c-plus-plus-api_dynamic-c-pluplus-wrapper.html). - -**2.** Создайте в любом месте новую директорию с произвольным названием, например `data` и поместите в нее созданную библиотеку. Docker-образ уже содержит библиотеку `data/libcatboostmodel.so`. - -**3.** Создайте в любом месте новую директорию для конфигурации модели с произвольным названием, например `models`. - -**4.** Создайте файл конфигурации модели с произвольным названием, например `models/amazon_model.xml`. - -**5.** Опишите конфигурацию модели: - -``` xml - - - - catboost - - amazon - - /home/catboost/tutorial/catboost_model.bin - - 0 - - -``` - -**6.** Добавьте в конфигурацию ClickHouse путь к CatBoost и конфигурации модели: - -``` xml - -/home/catboost/data/libcatboostmodel.so -/home/catboost/models/*_model.xml -``` - :::note "Примечание" - Вы можете позднее изменить путь к конфигурации модели CatBoost без перезагрузки сервера. - ::: -## 4. Запустите вывод модели из SQL {#run-model-inference} - -Для тестирования модели запустите клиент ClickHouse `$ clickhouse client`. - -Проверьте, что модель работает: - -``` sql -:) SELECT - modelEvaluate('amazon', - RESOURCE, - MGR_ID, - ROLE_ROLLUP_1, - ROLE_ROLLUP_2, - ROLE_DEPTNAME, - ROLE_TITLE, - ROLE_FAMILY_DESC, - ROLE_FAMILY, - ROLE_CODE) > 0 AS prediction, - ACTION AS target -FROM amazon_train -LIMIT 10 -``` - - :::note "Примечание" - Функция [modelEvaluate](../sql-reference/functions/other-functions.md#function-modelevaluate) возвращает кортежи (tuple) с исходными прогнозами по классам для моделей с несколькими классами. - ::: -Спрогнозируйте вероятность: - -``` sql -:) SELECT - modelEvaluate('amazon', - RESOURCE, - MGR_ID, - ROLE_ROLLUP_1, - ROLE_ROLLUP_2, - ROLE_DEPTNAME, - ROLE_TITLE, - ROLE_FAMILY_DESC, - ROLE_FAMILY, - ROLE_CODE) AS prediction, - 1. / (1 + exp(-prediction)) AS probability, - ACTION AS target -FROM amazon_train -LIMIT 10 -``` - - :::note "Примечание" - Подробнее про функцию [exp()](../sql-reference/functions/math-functions.md). - ::: -Посчитайте логистическую функцию потерь (LogLoss) на всей выборке: - -``` sql -:) SELECT -avg(tg * log(prob) + (1 - tg) * log(1 - prob)) AS logloss -FROM -( - SELECT - modelEvaluate('amazon', - RESOURCE, - MGR_ID, - ROLE_ROLLUP_1, - ROLE_ROLLUP_2, - ROLE_DEPTNAME, - ROLE_TITLE, - ROLE_FAMILY_DESC, - ROLE_FAMILY, - ROLE_CODE) AS prediction, - 1. / (1. + exp(-prediction)) AS prob, - ACTION AS tg - FROM amazon_train -) -``` - - :::note "Примечание" - Подробнее про функции [avg()](../sql-reference/aggregate-functions/reference/avg.md#agg_function-avg), [log()](../sql-reference/functions/math-functions.md). - ::: \ No newline at end of file diff --git a/docs/ru/guides/index.md b/docs/ru/guides/index.md index 0b5938dfc09..882f71b5700 100644 --- a/docs/ru/guides/index.md +++ b/docs/ru/guides/index.md @@ -7,5 +7,3 @@ sidebar_label: "Руководства" # Руководства {#rukovodstva} Подробные пошаговые инструкции, которые помогут вам решать различные задачи с помощью ClickHouse. - -- [Применение модели CatBoost в ClickHouse](apply-catboost-model.md) diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md index 5ddc684ce2a..0d4f0c63210 100644 --- a/docs/ru/operations/settings/settings.md +++ b/docs/ru/operations/settings/settings.md @@ -3799,6 +3799,17 @@ Exception: Total regexp lengths too large. Значение по умолчанию: `1`. +## enable_extended_results_for_datetime_functions {#enable-extended-results-for-datetime-functions} + +Включает или отключает возвращение результатов типа `Date32` с расширенным диапазоном (по сравнению с типом `Date`) для функций [toStartOfYear](../../sql-reference/functions/date-time-functions.md#tostartofyear), [toStartOfISOYear](../../sql-reference/functions/date-time-functions.md#tostartofisoyear), [toStartOfQuarter](../../sql-reference/functions/date-time-functions.md#tostartofquarter), [toStartOfMonth](../../sql-reference/functions/date-time-functions.md#tostartofmonth), [toStartOfWeek](../../sql-reference/functions/date-time-functions.md#tostartofweek), [toMonday](../../sql-reference/functions/date-time-functions.md#tomonday) и [toLastDayOfMonth](../../sql-reference/functions/date-time-functions.md#tolastdayofmonth). + +Возможные значения: + +- 0 — Функции возвращают результаты типа `Date` для всех типов аргументов. +- 1 — Функции возвращают результаты типа `Date32` для аргументов типа `Date32` или `DateTime64` и возвращают `Date` в других случаях. + +Значение по умолчанию: `0`. + **Пример** Запрос: diff --git a/docs/ru/sql-reference/functions/date-time-functions.md b/docs/ru/sql-reference/functions/date-time-functions.md index 1c623cd1dab..27689426cbe 100644 --- a/docs/ru/sql-reference/functions/date-time-functions.md +++ b/docs/ru/sql-reference/functions/date-time-functions.md @@ -268,24 +268,18 @@ SELECT toUnixTimestamp('2017-11-05 08:07:47', 'Asia/Tokyo') AS unix_timestamp; ``` :::note -Тип возвращаемого описанными далее функциями `toStartOf*`, `toMonday` значения - `Date` или `DateTime`. -Хотя эти функции могут принимать значения типа `Date32` или `DateTime64` в качестве аргумента, при обработке аргумента вне нормального диапазона значений (`1970` - `2148` для `Date` и `1970-01-01 00:00:00`-`2106-02-07 08:28:15` для `DateTime`) будет получен некорректный результат. -Возвращаемые значения для значений вне нормального диапазона: -* `1970-01-01 (00:00:00)` будет возвращён для моментов времени до 1970 года, -* `2106-02-07 08:28:15` будет взят в качестве аргумента, если полученный аргумент превосходит данное значение и возвращаемый тип - `DateTime`, -* `2149-06-06` будет взят в качестве аргумента, если полученный аргумент превосходит данное значение и возвращаемый тип - `Date`, -* `2149-05-31` будет результатом функции `toLastDayOfMonth` при обработке аргумента больше `2149-05-31`. +Тип возвращаемого значения описанными далее функциями `toStartOf*`, `toLastDayOfMonth`, `toMonday` определяется конфигурационным параметром [enable_extended_results_for_datetime_functions](../../operations/settings/settings#enable-extended-results-for-datetime-functions) имеющим по умолчанию значение `0`. + +Поведение для +* `enable_extended_results_for_datetime_functions = 0`: Функции `toStartOf*`, `toLastDayOfMonth`, `toMonday` возвращают `Date` или `DateTime`. Хотя эти функции могут принимать значения типа `Date32` или `DateTime64` в качестве аргумента, при обработке аргумента вне нормального диапазона значений (`1970` - `2148` для `Date` и `1970-01-01 00:00:00`-`2106-02-07 08:28:15` для `DateTime`) будет получен некорректный результат. +В случае если значение аргумента вне нормального диапазона: + * `1970-01-01 (00:00:00)` будет возвращён для моментов времени до 1970 года, + * `2106-02-07 08:28:15` будет взят в качестве аргумента, если полученный аргумент превосходит данное значение и возвращаемый тип - `DateTime`, + * `2149-06-06` будет взят в качестве аргумента, если полученный аргумент превосходит данное значение и возвращаемый тип - `Date`, + * `2149-05-31` будет результатом функции `toLastDayOfMonth` при обработке аргумента больше `2149-05-31`. +* `enable_extended_results_for_datetime_functions = 1`: Функции `toStartOfYear`, `toStartOfISOYear`, `toStartOfQuarter`, `toStartOfMonth`, `toStartOfWeek`, `toLastDayOfMonth`, `toMonday` возвращают `Date` или `DateTime` если их аргумент `Date` или `DateTime` и они возвращают `Date32` или `DateTime64` если их аргумент `Date32` или `DateTime64`. ::: -:::note -Тип возвращаемого описанными далее функциями `toStartOf*`, `toLastDayOfMonth`, `toMonday` значения - `Date` или `DateTime`. -Хотя эти функции могут принимать значения типа `Date32` или `DateTime64` в качестве аргумента, при обработке аргумента вне нормального диапазона значений (`1970` - `2148` для `Date` и `1970-01-01 00:00:00`-`2106-02-07 08:28:15` для `DateTime`) будет получен некорректный результат. -Возвращаемые значения для значений вне нормального диапазона: -* `1970-01-01 (00:00:00)` будет возвращён для моментов времени до 1970 года, -* `2106-02-07 08:28:15` будет взят в качестве аргумента, если полученный аргумент превосходит данное значение и возвращаемый тип - `DateTime`, -* `2149-06-06` будет взят в качестве аргумента, если полученный аргумент превосходит данное значение и возвращаемый тип - `Date`. - ::: -* ## toStartOfYear {#tostartofyear} Округляет дату или дату-с-временем вниз до первого дня года. @@ -324,6 +318,8 @@ SELECT toStartOfISOYear(toDate('2017-01-01')) AS ISOYear20170101; Округляет дату или дату-с-временем до последнего числа месяца. Возвращается дата. +Если `toLastDayOfMonth` вызывается с аргументом типа `Date` большим чем 2149-05-31, то результат будет вычислен от аргумента 2149-05-31. + ## toMonday {#tomonday} Округляет дату или дату-с-временем вниз до ближайшего понедельника. @@ -977,7 +973,7 @@ SELECT now('Europe/Moscow'); ## timeSlots(StartTime, Duration,\[, Size\]) {#timeslotsstarttime-duration-size} Для интервала, начинающегося в `StartTime` и длящегося `Duration` секунд, возвращает массив моментов времени, кратных `Size`. Параметр `Size` указывать необязательно, по умолчанию он равен 1800 секундам (30 минутам) - необязательный параметр. Данная функция может использоваться, например, для анализа количества просмотров страницы за соответствующую сессию. -Аргумент `StartTime` может иметь тип `DateTime` или `DateTime64`. В случае, если используется `DateTime`, аргументы `Duration` и `Size` должны иметь тип `UInt32`; Для DateTime64 они должны быть типа `Decimal64`. +Аргумент `StartTime` может иметь тип `DateTime` или `DateTime64`. В случае, если используется `DateTime`, аргументы `Duration` и `Size` должны иметь тип `UInt32`; Для DateTime64 они должны быть типа `Decimal64`. Возвращает массив DateTime/DateTime64 (тип будет совпадать с типом параметра ’StartTime’). Для DateTime64 масштаб(scale) возвращаемой величины может отличаться от масштаба фргумента ’StartTime’ --- результат будет иметь наибольший масштаб среди всех данных аргументов. Пример использования: diff --git a/docs/ru/sql-reference/statements/system.md b/docs/ru/sql-reference/statements/system.md index c1dc03a63d1..a7dec7abe27 100644 --- a/docs/ru/sql-reference/statements/system.md +++ b/docs/ru/sql-reference/statements/system.md @@ -6,43 +6,6 @@ sidebar_label: SYSTEM # Запросы SYSTEM {#query-language-system} -- [RELOAD EMBEDDED DICTIONARIES](#query_language-system-reload-emdedded-dictionaries) -- [RELOAD DICTIONARIES](#query_language-system-reload-dictionaries) -- [RELOAD DICTIONARY](#query_language-system-reload-dictionary) -- [RELOAD MODELS](#query_language-system-reload-models) -- [RELOAD MODEL](#query_language-system-reload-model) -- [RELOAD FUNCTIONS](#query_language-system-reload-functions) -- [RELOAD FUNCTION](#query_language-system-reload-functions) -- [DROP DNS CACHE](#query_language-system-drop-dns-cache) -- [DROP MARK CACHE](#query_language-system-drop-mark-cache) -- [DROP UNCOMPRESSED CACHE](#query_language-system-drop-uncompressed-cache) -- [DROP COMPILED EXPRESSION CACHE](#query_language-system-drop-compiled-expression-cache) -- [DROP REPLICA](#query_language-system-drop-replica) -- [FLUSH LOGS](#query_language-system-flush_logs) -- [RELOAD CONFIG](#query_language-system-reload-config) -- [SHUTDOWN](#query_language-system-shutdown) -- [KILL](#query_language-system-kill) -- [STOP DISTRIBUTED SENDS](#query_language-system-stop-distributed-sends) -- [FLUSH DISTRIBUTED](#query_language-system-flush-distributed) -- [START DISTRIBUTED SENDS](#query_language-system-start-distributed-sends) -- [STOP MERGES](#query_language-system-stop-merges) -- [START MERGES](#query_language-system-start-merges) -- [STOP TTL MERGES](#query_language-stop-ttl-merges) -- [START TTL MERGES](#query_language-start-ttl-merges) -- [STOP MOVES](#query_language-stop-moves) -- [START MOVES](#query_language-start-moves) -- [SYSTEM UNFREEZE](#query_language-system-unfreeze) -- [STOP FETCHES](#query_language-system-stop-fetches) -- [START FETCHES](#query_language-system-start-fetches) -- [STOP REPLICATED SENDS](#query_language-system-start-replicated-sends) -- [START REPLICATED SENDS](#query_language-system-start-replicated-sends) -- [STOP REPLICATION QUEUES](#query_language-system-stop-replication-queues) -- [START REPLICATION QUEUES](#query_language-system-start-replication-queues) -- [SYNC REPLICA](#query_language-system-sync-replica) -- [RESTART REPLICA](#query_language-system-restart-replica) -- [RESTORE REPLICA](#query_language-system-restore-replica) -- [RESTART REPLICAS](#query_language-system-restart-replicas) - ## RELOAD EMBEDDED DICTIONARIES] {#query_language-system-reload-emdedded-dictionaries} Перегружает все [Встроенные словари](../dictionaries/internal-dicts.md). По умолчанию встроенные словари выключены. @@ -66,7 +29,12 @@ SELECT name, status FROM system.dictionaries; ## RELOAD MODELS {#query_language-system-reload-models} -Перегружает все модели [CatBoost](../../guides/apply-catboost-model.md#applying-catboost-model-in-clickhouse), если их конфигурация была обновлена, без перезагрузки сервера. +:::note +Это утверждение и `SYSTEM RELOAD MODEL` просто выгружают модели catboost из clickhouse-library-bridge. Функция `catboostEvaluate()` +загружает модель при первом обращении, если она еще не загружена. +::: + +Разгрузите все модели CatBoost. **Синтаксис** @@ -76,12 +44,12 @@ SYSTEM RELOAD MODELS ## RELOAD MODEL {#query_language-system-reload-model} -Полностью перегружает модель [CatBoost](../../guides/apply-catboost-model.md#applying-catboost-model-in-clickhouse) `model_name`, если ее конфигурация была обновлена, без перезагрузки сервера. +Выгружает модель CatBoost по адресу `модель_путь`. **Синтаксис** ```sql -SYSTEM RELOAD MODEL +SYSTEM RELOAD MODEL ``` ## RELOAD FUNCTIONS {#query_language-system-reload-functions} diff --git a/docs/ru/sql-reference/table-functions/file.md b/docs/ru/sql-reference/table-functions/file.md index 1f262c9403a..df35a1c4ac0 100644 --- a/docs/ru/sql-reference/table-functions/file.md +++ b/docs/ru/sql-reference/table-functions/file.md @@ -13,7 +13,7 @@ sidebar_label: file **Синтаксис** ``` sql -file(path, format, structure) +file(path [,format] [,structure]) ``` **Параметры** diff --git a/docs/ru/sql-reference/table-functions/s3.md b/docs/ru/sql-reference/table-functions/s3.md index ae0419a4b84..14c8204fd1d 100644 --- a/docs/ru/sql-reference/table-functions/s3.md +++ b/docs/ru/sql-reference/table-functions/s3.md @@ -11,7 +11,7 @@ sidebar_label: s3 **Синтаксис** ``` sql -s3(path, [aws_access_key_id, aws_secret_access_key,] format, structure, [compression]) +s3(path [,aws_access_key_id, aws_secret_access_key] [,format] [,structure] [,compression]) ``` **Aргументы** diff --git a/docs/ru/sql-reference/table-functions/s3Cluster.md b/docs/ru/sql-reference/table-functions/s3Cluster.md index e6b317253c0..1c12913fabe 100644 --- a/docs/ru/sql-reference/table-functions/s3Cluster.md +++ b/docs/ru/sql-reference/table-functions/s3Cluster.md @@ -11,7 +11,7 @@ sidebar_label: s3Cluster **Синтаксис** ``` sql -s3Cluster(cluster_name, source, [access_key_id, secret_access_key,] format, structure) +s3Cluster(cluster_name, source, [,access_key_id, secret_access_key] [,format] [,structure]) ``` **Аргументы** diff --git a/docs/ru/sql-reference/table-functions/url.md b/docs/ru/sql-reference/table-functions/url.md index d4fb11b0de7..e5d9faeec00 100644 --- a/docs/ru/sql-reference/table-functions/url.md +++ b/docs/ru/sql-reference/table-functions/url.md @@ -13,7 +13,7 @@ sidebar_label: url **Синтаксис** ``` sql -url(URL, format, structure) +url(URL [,format] [,structure]) ``` **Параметры** diff --git a/docs/zh/guides/apply-catboost-model.md b/docs/zh/guides/apply-catboost-model.md deleted file mode 100644 index 861e5372875..00000000000 --- a/docs/zh/guides/apply-catboost-model.md +++ /dev/null @@ -1,244 +0,0 @@ ---- -slug: /zh/guides/apply-catboost-model -sidebar_position: 41 -sidebar_label: "\u5E94\u7528CatBoost\u6A21\u578B" ---- - -# 在ClickHouse中应用Catboost模型 {#applying-catboost-model-in-clickhouse} - -[CatBoost](https://catboost.ai) 是一个由[Yandex](https://yandex.com/company/)开发的开源免费机器学习库。 - - -通过本篇文档，您将学会如何用SQL语句调用已经存放在Clickhouse中的预训练模型来预测数据。 - - -为了在ClickHouse中应用CatBoost模型，需要进行如下步骤： - -1. [创建数据表](#create-table). -2. [将数据插入到表中](#insert-data-to-table). -3. [将CatBoost集成到ClickHouse中](#integrate-catboost-into-clickhouse) （可跳过）。 -4. [从SQL运行模型推断](#run-model-inference). - -有关训练CatBoost模型的详细信息，请参阅 [训练和模型应用](https://catboost.ai/docs/features/training.html#training). - -您可以通过[RELOAD MODEL](https://clickhouse.com/docs/en/sql-reference/statements/system/#query_language-system-reload-model)与[RELOAD MODELS](https://clickhouse.com/docs/en/sql-reference/statements/system/#query_language-system-reload-models)语句来重载CatBoost模型。 - -## 先决条件 {#prerequisites} - -请先安装 [Docker](https://docs.docker.com/install/)。 - -!!! note "注" - [Docker](https://www.docker.com) 是一个软件平台，用户可以用Docker来创建独立于已有系统并集成了CatBoost和ClickHouse的容器。 - -在应用CatBoost模型之前: - -**1.** 从容器仓库拉取示例docker镜像 (https://hub.docker.com/r/yandex/tutorial-catboost-clickhouse) : - -``` bash -$ docker pull yandex/tutorial-catboost-clickhouse -``` - -此示例Docker镜像包含运行CatBoost和ClickHouse所需的所有内容：代码、运行时、库、环境变量和配置文件。 - -**2.** 确保已成功拉取Docker镜像: - -``` bash -$ docker image ls -REPOSITORY TAG IMAGE ID CREATED SIZE -yandex/tutorial-catboost-clickhouse latest 622e4d17945b 22 hours ago 1.37GB -``` - -**3.** 基于此镜像启动一个Docker容器: - -``` bash -$ docker run -it -p 8888:8888 yandex/tutorial-catboost-clickhouse -``` - -## 1. 创建数据表 {#create-table} - -为训练样本创建ClickHouse表: - -**1.** 在交互模式下启动ClickHouse控制台客户端: - -``` bash -$ clickhouse client -``` - -!!! note "注" - ClickHouse服务器已经在Docker容器内运行。 - -**2.** 使用以下命令创建表: - -``` sql -:) CREATE TABLE amazon_train -( - date Date MATERIALIZED today(), - ACTION UInt8, - RESOURCE UInt32, - MGR_ID UInt32, - ROLE_ROLLUP_1 UInt32, - ROLE_ROLLUP_2 UInt32, - ROLE_DEPTNAME UInt32, - ROLE_TITLE UInt32, - ROLE_FAMILY_DESC UInt32, - ROLE_FAMILY UInt32, - ROLE_CODE UInt32 -) -ENGINE = MergeTree ORDER BY date -``` - -**3.** 从ClickHouse控制台客户端退出: - -``` sql -:) exit -``` - -## 2. 将数据插入到表中 {#insert-data-to-table} - -插入数据: - -**1.** 运行以下命令: - -``` bash -$ clickhouse client --host 127.0.0.1 --query 'INSERT INTO amazon_train FORMAT CSVWithNames' < ~/amazon/train.csv -``` - -**2.** 在交互模式下启动ClickHouse控制台客户端: - -``` bash -$ clickhouse client -``` - -**3.** 确保数据已上传: - -``` sql -:) SELECT count() FROM amazon_train - -SELECT count() -FROM amazon_train - -+-count()-+ -| 65538 | -+-------+ -``` - -## 3. 将CatBoost集成到ClickHouse中 {#integrate-catboost-into-clickhouse} - -!!! note "注" - **可跳过。** 示例Docker映像已经包含了运行CatBoost和ClickHouse所需的所有内容。 - -为了将CatBoost集成进ClickHouse，需要进行如下步骤： - -**1.** 构建评估库。 - -评估CatBoost模型的最快方法是编译 `libcatboostmodel.` 库文件. - -有关如何构建库文件的详细信息，请参阅 [CatBoost文件](https://catboost.ai/docs/concepts/c-plus-plus-api_dynamic-c-pluplus-wrapper.html). - -**2.** 创建一个新目录（位置与名称可随意指定）, 如 `data` 并将创建的库文件放入其中。 示例Docker镜像已经包含了库 `data/libcatboostmodel.so`. - -**3.** 创建一个新目录来放配置模型, 如 `models`. - -**4.** 创建一个模型配置文件，如 `models/amazon_model.xml`. - -**5.** 修改模型配置: - -``` xml - - - - catboost - - amazon - - /home/catboost/tutorial/catboost_model.bin - - 0 - - -``` - -**6.** 将CatBoost库文件的路径和模型配置添加到ClickHouse配置: - -``` xml - -/home/catboost/data/libcatboostmodel.so -/home/catboost/models/*_model.xml -``` - -## 4. 使用SQL调用预测模型 {#run-model-inference} - -为了测试模型是否正常，可以使用ClickHouse客户端 `$ clickhouse client`. - -让我们确保模型能正常工作: - -``` sql -:) SELECT - modelEvaluate('amazon', - RESOURCE, - MGR_ID, - ROLE_ROLLUP_1, - ROLE_ROLLUP_2, - ROLE_DEPTNAME, - ROLE_TITLE, - ROLE_FAMILY_DESC, - ROLE_FAMILY, - ROLE_CODE) > 0 AS prediction, - ACTION AS target -FROM amazon_train -LIMIT 10 -``` - -!!! note "注" - 函数 [modelEvaluate](../sql-reference/functions/other-functions.md#function-modelevaluate) 会对多类别模型返回一个元组，其中包含每一类别的原始预测值。 - -执行预测: - -``` sql -:) SELECT - modelEvaluate('amazon', - RESOURCE, - MGR_ID, - ROLE_ROLLUP_1, - ROLE_ROLLUP_2, - ROLE_DEPTNAME, - ROLE_TITLE, - ROLE_FAMILY_DESC, - ROLE_FAMILY, - ROLE_CODE) AS prediction, - 1. / (1 + exp(-prediction)) AS probability, - ACTION AS target -FROM amazon_train -LIMIT 10 -``` - -!!! note "注" - 查看函数说明 [exp()](../sql-reference/functions/math-functions.md) 。 - -让我们计算样本的LogLoss: - -``` sql -:) SELECT -avg(tg * log(prob) + (1 - tg) * log(1 - prob)) AS logloss -FROM -( - SELECT - modelEvaluate('amazon', - RESOURCE, - MGR_ID, - ROLE_ROLLUP_1, - ROLE_ROLLUP_2, - ROLE_DEPTNAME, - ROLE_TITLE, - ROLE_FAMILY_DESC, - ROLE_FAMILY, - ROLE_CODE) AS prediction, - 1. / (1. + exp(-prediction)) AS prob, - ACTION AS tg - FROM amazon_train -) -``` - -!!! note "注" - 查看函数说明 [avg()](../sql-reference/aggregate-functions/reference/avg.md#agg_function-avg) 和 [log()](../sql-reference/functions/math-functions.md) 。 - -[原始文章](https://clickhouse.com/docs/en/guides/apply_catboost_model/) diff --git a/docs/zh/guides/index.md b/docs/zh/guides/index.md index 5e535ea5736..00c4ae4def1 100644 --- a/docs/zh/guides/index.md +++ b/docs/zh/guides/index.md @@ -9,6 +9,5 @@ sidebar_label: ClickHouse指南 列出了如何使用 Clickhouse 解决各种任务的详细说明： - [关于简单集群设置的教程](../getting-started/tutorial.md) -- [在ClickHouse中应用CatBoost模型](apply-catboost-model.md) [原始文章](https://clickhouse.com/docs/en/guides/) diff --git a/docs/zh/sql-reference/statements/system.md b/docs/zh/sql-reference/statements/system.md index d833887a9c6..3df00cf8854 100644 --- a/docs/zh/sql-reference/statements/system.md +++ b/docs/zh/sql-reference/statements/system.md @@ -6,38 +6,6 @@ sidebar_label: SYSTEM # SYSTEM Queries {#query-language-system} -- [RELOAD EMBEDDED DICTIONARIES](#query_language-system-reload-emdedded-dictionaries) -- [RELOAD DICTIONARIES](#query_language-system-reload-dictionaries) -- [RELOAD DICTIONARY](#query_language-system-reload-dictionary) -- [DROP DNS CACHE](#query_language-system-drop-dns-cache) -- [DROP MARK CACHE](#query_language-system-drop-mark-cache) -- [DROP UNCOMPRESSED CACHE](#query_language-system-drop-uncompressed-cache) -- [DROP COMPILED EXPRESSION CACHE](#query_language-system-drop-compiled-expression-cache) -- [DROP REPLICA](#query_language-system-drop-replica) -- [FLUSH LOGS](#query_language-system-flush_logs) -- [RELOAD CONFIG](#query_language-system-reload-config) -- [SHUTDOWN](#query_language-system-shutdown) -- [KILL](#query_language-system-kill) -- [STOP DISTRIBUTED SENDS](#query_language-system-stop-distributed-sends) -- [FLUSH DISTRIBUTED](#query_language-system-flush-distributed) -- [START DISTRIBUTED SENDS](#query_language-system-start-distributed-sends) -- [STOP MERGES](#query_language-system-stop-merges) -- [START MERGES](#query_language-system-start-merges) -- [STOP TTL MERGES](#query_language-stop-ttl-merges) -- [START TTL MERGES](#query_language-start-ttl-merges) -- [STOP MOVES](#query_language-stop-moves) -- [START MOVES](#query_language-start-moves) -- [SYSTEM UNFREEZE](#query_language-system-unfreeze) -- [STOP FETCHES](#query_language-system-stop-fetches) -- [START FETCHES](#query_language-system-start-fetches) -- [STOP REPLICATED SENDS](#query_language-system-start-replicated-sends) -- [START REPLICATED SENDS](#query_language-system-start-replicated-sends) -- [STOP REPLICATION QUEUES](#query_language-system-stop-replication-queues) -- [START REPLICATION QUEUES](#query_language-system-start-replication-queues) -- [SYNC REPLICA](#query_language-system-sync-replica) -- [RESTART REPLICA](#query_language-system-restart-replica) -- [RESTART REPLICAS](#query_language-system-restart-replicas) - ## RELOAD EMBEDDED DICTIONARIES\] {#query_language-system-reload-emdedded-dictionaries} 重新加载所有[内置字典](../../sql-reference/dictionaries/internal-dicts.md)。默认情况下内置字典是禁用的。 diff --git a/packages/clickhouse-server.init b/packages/clickhouse-server.init index 1695f6286b8..13aeffe13a7 100755 --- a/packages/clickhouse-server.init +++ b/packages/clickhouse-server.init @@ -47,9 +47,10 @@ CLICKHOUSE_PIDFILE="$CLICKHOUSE_PIDDIR/$PROGRAM.pid" # Some systems lack "flock" command -v flock >/dev/null && FLOCK=flock -# Override defaults from optional config file +# Override defaults from optional config file and export them automatically +set -a test -f /etc/default/clickhouse && . /etc/default/clickhouse - +set +a die() { diff --git a/programs/CMakeLists.txt b/programs/CMakeLists.txt index 3c0c0781de6..9cf7cb2b624 100644 --- a/programs/CMakeLists.txt +++ b/programs/CMakeLists.txt @@ -54,7 +54,7 @@ else () endif () if (NOT USE_MUSL) - option (ENABLE_CLICKHOUSE_LIBRARY_BRIDGE "HTTP-server working like a proxy to Library dictionary source" ${ENABLE_CLICKHOUSE_ALL}) + option (ENABLE_CLICKHOUSE_LIBRARY_BRIDGE "HTTP-server working like a proxy to external dynamically loaded libraries" ${ENABLE_CLICKHOUSE_ALL}) endif () # https://presentations.clickhouse.com/matemarketing_2020/ diff --git a/programs/benchmark/clickhouse-benchmark.cpp b/programs/benchmark/clickhouse-benchmark.cpp index 6bcb6e19b88..d58dd4dfbe7 100644 --- a/programs/benchmark/clickhouse-benchmark.cpp +++ b/programs/benchmark/clickhouse-benchmark.cpp @@ -1,2 +1 @@ extern int mainEntryClickHouseBenchmark(int argc, char ** argv); -int main(int argc_, char ** argv_) { return mainEntryClickHouseBenchmark(argc_, argv_); } diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index 6506c23428a..5bd9d28d8e3 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -150,7 +150,7 @@ std::vector Client::loadWarningMessages() size_t rows = packet.block.rows(); for (size_t i = 0; i < rows; ++i) - messages.emplace_back(column.getDataAt(i).toString()); + messages.emplace_back(column[i].get()); } continue; diff --git a/programs/client/clickhouse-client.cpp b/programs/client/clickhouse-client.cpp index 8f7da31d617..3e489c5ff15 100644 --- a/programs/client/clickhouse-client.cpp +++ b/programs/client/clickhouse-client.cpp @@ -1,2 +1 @@ int mainEntryClickHouseClient(int argc, char ** argv); -int main(int argc_, char ** argv_) { return mainEntryClickHouseClient(argc_, argv_); } diff --git a/programs/compressor/clickhouse-compressor.cpp b/programs/compressor/clickhouse-compressor.cpp index f7d8611eac5..13a00ab6aff 100644 --- a/programs/compressor/clickhouse-compressor.cpp +++ b/programs/compressor/clickhouse-compressor.cpp @@ -1,2 +1 @@ int mainEntryClickHouseCompressor(int argc, char ** argv); -int main(int argc_, char ** argv_) { return mainEntryClickHouseCompressor(argc_, argv_); } diff --git a/programs/copier/clickhouse-copier.cpp b/programs/copier/clickhouse-copier.cpp index 653a0128aa4..4dabb01775b 100644 --- a/programs/copier/clickhouse-copier.cpp +++ b/programs/copier/clickhouse-copier.cpp @@ -1,2 +1 @@ int mainEntryClickHouseClusterCopier(int argc, char ** argv); -int main(int argc_, char ** argv_) { return mainEntryClickHouseClusterCopier(argc_, argv_); } diff --git a/programs/disks/clickhouse-disks.cpp b/programs/disks/clickhouse-disks.cpp index 184de86ee77..67ace2ddfd3 100644 --- a/programs/disks/clickhouse-disks.cpp +++ b/programs/disks/clickhouse-disks.cpp @@ -1,2 +1 @@ int mainEntryClickHouseDisks(int argc, char ** argv); -int main(int argc_, char ** argv_) { return mainEntryClickHouseDisks(argc_, argv_); } diff --git a/programs/extract-from-config/clickhouse-extract-from-config.cpp b/programs/extract-from-config/clickhouse-extract-from-config.cpp index 95d119b0dad..9a384c8f2dc 100644 --- a/programs/extract-from-config/clickhouse-extract-from-config.cpp +++ b/programs/extract-from-config/clickhouse-extract-from-config.cpp @@ -1,2 +1 @@ int mainEntryClickHouseExtractFromConfig(int argc, char ** argv); -int main(int argc_, char ** argv_) { return mainEntryClickHouseExtractFromConfig(argc_, argv_); } diff --git a/programs/format/clickhouse-format.cpp b/programs/format/clickhouse-format.cpp index 6bda5c5e039..71f2a071312 100644 --- a/programs/format/clickhouse-format.cpp +++ b/programs/format/clickhouse-format.cpp @@ -1,2 +1 @@ int mainEntryClickHouseFormat(int argc, char ** argv); -int main(int argc_, char ** argv_) { return mainEntryClickHouseFormat(argc_, argv_); } diff --git a/programs/git-import/clickhouse-git-import.cpp b/programs/git-import/clickhouse-git-import.cpp index cfa06306604..57b66f3f77a 100644 --- a/programs/git-import/clickhouse-git-import.cpp +++ b/programs/git-import/clickhouse-git-import.cpp @@ -1,2 +1 @@ int mainEntryClickHouseGitImport(int argc, char ** argv); -int main(int argc_, char ** argv_) { return mainEntryClickHouseGitImport(argc_, argv_); } diff --git a/programs/install/Install.cpp b/programs/install/Install.cpp index 297e2d24c07..9b1bae947d2 100644 --- a/programs/install/Install.cpp +++ b/programs/install/Install.cpp @@ -446,8 +446,8 @@ int mainEntryClickHouseInstall(int argc, char ** argv) fs::path ulimits_file = ulimits_dir / fmt::format("{}.conf", user); fmt::print("Will set ulimits for {} user in {}.\n", user, ulimits_file.string()); std::string ulimits_content = fmt::format( - "{0}\tsoft\tnofile\t262144\n" - "{0}\thard\tnofile\t262144\n", user); + "{0}\tsoft\tnofile\t1048576\n" + "{0}\thard\tnofile\t1048576\n", user); fs::create_directories(ulimits_dir); diff --git a/programs/keeper-converter/clickhouse-keeper-converter.cpp b/programs/keeper-converter/clickhouse-keeper-converter.cpp index 3cb6f99f837..c84fc51f8cc 100644 --- a/programs/keeper-converter/clickhouse-keeper-converter.cpp +++ b/programs/keeper-converter/clickhouse-keeper-converter.cpp @@ -1,2 +1 @@ int mainEntryClickHouseKeeperConverter(int argc, char ** argv); -int main(int argc_, char ** argv_) { return mainEntryClickHouseKeeperConverter(argc_, argv_); } diff --git a/programs/keeper/CMakeLists.txt b/programs/keeper/CMakeLists.txt index 36e37eda91f..a5ad506abe6 100644 --- a/programs/keeper/CMakeLists.txt +++ b/programs/keeper/CMakeLists.txt @@ -103,6 +103,7 @@ if (BUILD_STANDALONE_KEEPER) # Remove some redundant dependencies target_compile_definitions (clickhouse-keeper PRIVATE -DKEEPER_STANDALONE_BUILD) + target_compile_definitions (clickhouse-keeper PUBLIC -DWITHOUT_TEXT_LOG) target_include_directories(clickhouse-keeper PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/../../src") # uses includes from src directory target_include_directories(clickhouse-keeper PUBLIC "${CMAKE_CURRENT_BINARY_DIR}/../../src/Core/include") # uses some includes from core diff --git a/programs/library-bridge/CMakeLists.txt b/programs/library-bridge/CMakeLists.txt index 40cabacded4..1cacc391ca5 100644 --- a/programs/library-bridge/CMakeLists.txt +++ b/programs/library-bridge/CMakeLists.txt @@ -1,12 +1,15 @@ include(${ClickHouse_SOURCE_DIR}/cmake/split_debug_symbols.cmake) set (CLICKHOUSE_LIBRARY_BRIDGE_SOURCES + CatBoostLibraryHandler.cpp + CatBoostLibraryHandlerFactory.cpp ExternalDictionaryLibraryAPI.cpp ExternalDictionaryLibraryHandler.cpp ExternalDictionaryLibraryHandlerFactory.cpp LibraryBridge.cpp LibraryBridgeHandlerFactory.cpp LibraryBridgeHandlers.cpp + SharedLibrary.cpp library-bridge.cpp ) diff --git a/programs/library-bridge/CatBoostLibraryAPI.h b/programs/library-bridge/CatBoostLibraryAPI.h new file mode 100644 index 00000000000..9eaa0d17ff7 --- /dev/null +++ b/programs/library-bridge/CatBoostLibraryAPI.h @@ -0,0 +1,49 @@ +#pragma once + +#include +#include + +// Function pointer typedefs and names of libcatboost.so functions used by ClickHouse +struct CatBoostLibraryAPI +{ + using ModelCalcerHandle = void; + + using ModelCalcerCreateFunc = ModelCalcerHandle * (*)(); + static constexpr const char * ModelCalcerCreateName = "ModelCalcerCreate"; + + using ModelCalcerDeleteFunc = void (*)(ModelCalcerHandle *); + static constexpr const char * ModelCalcerDeleteName = "ModelCalcerDelete"; + + using GetErrorStringFunc = const char * (*)(); + static constexpr const char * GetErrorStringName = "GetErrorString"; + + using LoadFullModelFromFileFunc = bool (*)(ModelCalcerHandle *, const char *); + static constexpr const char * LoadFullModelFromFileName = "LoadFullModelFromFile"; + + using CalcModelPredictionFlatFunc = bool (*)(ModelCalcerHandle *, size_t, const float **, size_t, double *, size_t); + static constexpr const char * CalcModelPredictionFlatName = "CalcModelPredictionFlat"; + + using CalcModelPredictionFunc = bool (*)(ModelCalcerHandle *, size_t, const float **, size_t, const char ***, size_t, double *, size_t); + static constexpr const char * CalcModelPredictionName = "CalcModelPrediction"; + + using CalcModelPredictionWithHashedCatFeaturesFunc = bool (*)(ModelCalcerHandle *, size_t, const float **, size_t, const int **, size_t, double *, size_t); + static constexpr const char * CalcModelPredictionWithHashedCatFeaturesName = "CalcModelPredictionWithHashedCatFeatures"; + + using GetStringCatFeatureHashFunc = int (*)(const char *, size_t); + static constexpr const char * GetStringCatFeatureHashName = "GetStringCatFeatureHash"; + + using GetIntegerCatFeatureHashFunc = int (*)(uint64_t); + static constexpr const char * GetIntegerCatFeatureHashName = "GetIntegerCatFeatureHash"; + + using GetFloatFeaturesCountFunc = size_t (*)(ModelCalcerHandle *); + static constexpr const char * GetFloatFeaturesCountName = "GetFloatFeaturesCount"; + + using GetCatFeaturesCountFunc = size_t (*)(ModelCalcerHandle *); + static constexpr const char * GetCatFeaturesCountName = "GetCatFeaturesCount"; + + using GetTreeCountFunc = size_t (*)(ModelCalcerHandle *); + static constexpr const char * GetTreeCountName = "GetTreeCount"; + + using GetDimensionsCountFunc = size_t (*)(ModelCalcerHandle *); + static constexpr const char * GetDimensionsCountName = "GetDimensionsCount"; +}; diff --git a/programs/library-bridge/CatBoostLibraryHandler.cpp b/programs/library-bridge/CatBoostLibraryHandler.cpp new file mode 100644 index 00000000000..4fe539a53b2 --- /dev/null +++ b/programs/library-bridge/CatBoostLibraryHandler.cpp @@ -0,0 +1,389 @@ +#include "CatBoostLibraryHandler.h" + +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int BAD_ARGUMENTS; + extern const int CANNOT_APPLY_CATBOOST_MODEL; + extern const int CANNOT_LOAD_CATBOOST_MODEL; + extern const int LOGICAL_ERROR; +} + +CatBoostLibraryHandler::APIHolder::APIHolder(SharedLibrary & lib) +{ + ModelCalcerCreate = lib.get(CatBoostLibraryAPI::ModelCalcerCreateName); + ModelCalcerDelete = lib.get(CatBoostLibraryAPI::ModelCalcerDeleteName); + GetErrorString = lib.get(CatBoostLibraryAPI::GetErrorStringName); + LoadFullModelFromFile = lib.get(CatBoostLibraryAPI::LoadFullModelFromFileName); + CalcModelPredictionFlat = lib.get(CatBoostLibraryAPI::CalcModelPredictionFlatName); + CalcModelPrediction = lib.get(CatBoostLibraryAPI::CalcModelPredictionName); + CalcModelPredictionWithHashedCatFeatures = lib.get(CatBoostLibraryAPI::CalcModelPredictionWithHashedCatFeaturesName); + GetStringCatFeatureHash = lib.get(CatBoostLibraryAPI::GetStringCatFeatureHashName); + GetIntegerCatFeatureHash = lib.get(CatBoostLibraryAPI::GetIntegerCatFeatureHashName); + GetFloatFeaturesCount = lib.get(CatBoostLibraryAPI::GetFloatFeaturesCountName); + GetCatFeaturesCount = lib.get(CatBoostLibraryAPI::GetCatFeaturesCountName); + GetTreeCount = lib.tryGet(CatBoostLibraryAPI::GetTreeCountName); + GetDimensionsCount = lib.tryGet(CatBoostLibraryAPI::GetDimensionsCountName); +} + +CatBoostLibraryHandler::CatBoostLibraryHandler( + const std::string & library_path, + const std::string & model_path) + : loading_start_time(std::chrono::system_clock::now()) + , library(std::make_shared(library_path)) + , api(*library) +{ + model_calcer_handle = api.ModelCalcerCreate(); + + if (!api.LoadFullModelFromFile(model_calcer_handle, model_path.c_str())) + { + throw Exception(ErrorCodes::CANNOT_LOAD_CATBOOST_MODEL, + "Cannot load CatBoost model: {}", api.GetErrorString()); + } + + float_features_count = api.GetFloatFeaturesCount(model_calcer_handle); + cat_features_count = api.GetCatFeaturesCount(model_calcer_handle); + + tree_count = 1; + if (api.GetDimensionsCount) + tree_count = api.GetDimensionsCount(model_calcer_handle); + + loading_duration = std::chrono::duration_cast(std::chrono::system_clock::now() - loading_start_time); +} + +CatBoostLibraryHandler::~CatBoostLibraryHandler() +{ + api.ModelCalcerDelete(model_calcer_handle); +} + +std::chrono::system_clock::time_point CatBoostLibraryHandler::getLoadingStartTime() const +{ + return loading_start_time; +} + +std::chrono::milliseconds CatBoostLibraryHandler::getLoadingDuration() const +{ + return loading_duration; +} + +namespace +{ + +/// Buffer should be allocated with features_count * column->size() elements. +/// Place column elements in positions buffer[0], buffer[features_count], ... , buffer[size * features_count] +template +void placeColumnAsNumber(const IColumn * column, T * buffer, size_t features_count) +{ + size_t size = column->size(); + FieldVisitorConvertToNumber visitor; + for (size_t i = 0; i < size; ++i) + { + /// TODO: Replace with column visitor. + Field field; + column->get(i, field); + *buffer = applyVisitor(visitor, field); + buffer += features_count; + } +} + +/// Buffer should be allocated with features_count * column->size() elements. +/// Place string pointers in positions buffer[0], buffer[features_count], ... , buffer[size * features_count] +void placeStringColumn(const ColumnString & column, const char ** buffer, size_t features_count) +{ + size_t size = column.size(); + for (size_t i = 0; i < size; ++i) + { + *buffer = const_cast(column.getDataAt(i).data); + buffer += features_count; + } +} + +/// Buffer should be allocated with features_count * column->size() elements. +/// Place string pointers in positions buffer[0], buffer[features_count], ... , buffer[size * features_count] +/// Returns PODArray which holds data (because ColumnFixedString doesn't store terminating zero). +PODArray placeFixedStringColumn(const ColumnFixedString & column, const char ** buffer, size_t features_count) +{ + size_t size = column.size(); + size_t str_size = column.getN(); + PODArray data(size * (str_size + 1)); + char * data_ptr = data.data(); + + for (size_t i = 0; i < size; ++i) + { + auto ref = column.getDataAt(i); + memcpy(data_ptr, ref.data, ref.size); + data_ptr[ref.size] = 0; + *buffer = data_ptr; + data_ptr += ref.size + 1; + buffer += features_count; + } + + return data; +} + +/// Place columns into buffer, returns column which holds placed data. Buffer should contains column->size() values. +template +ColumnPtr placeNumericColumns(const ColumnRawPtrs & columns, size_t offset, size_t size, const T** buffer) +{ + if (size == 0) + return nullptr; + + size_t column_size = columns[offset]->size(); + auto data_column = ColumnVector::create(size * column_size); + T * data = data_column->getData().data(); + for (size_t i = 0; i < size; ++i) + { + const auto * column = columns[offset + i]; + if (column->isNumeric()) + placeColumnAsNumber(column, data + i, size); + } + + for (size_t i = 0; i < column_size; ++i) + { + *buffer = data; + ++buffer; + data += size; + } + + return data_column; +} + +/// Place columns into buffer, returns data which was used for fixed string columns. +/// Buffer should contains column->size() values, each value contains size strings. +std::vector> placeStringColumns(const ColumnRawPtrs & columns, size_t offset, size_t size, const char ** buffer) +{ + if (size == 0) + return {}; + + std::vector> data; + for (size_t i = 0; i < size; ++i) + { + const auto * column = columns[offset + i]; + if (const auto * column_string = typeid_cast(column)) + placeStringColumn(*column_string, buffer + i, size); + else if (const auto * column_fixed_string = typeid_cast(column)) + data.push_back(placeFixedStringColumn(*column_fixed_string, buffer + i, size)); + else + throw Exception("Cannot place string column.", ErrorCodes::LOGICAL_ERROR); + } + + return data; +} + +/// buffer[column_size * cat_features_count] -> char * => cat_features[column_size][cat_features_count] -> char * +void fillCatFeaturesBuffer( + const char *** cat_features, const char ** buffer, + size_t column_size, size_t cat_features_count) +{ + for (size_t i = 0; i < column_size; ++i) + { + *cat_features = buffer; + ++cat_features; + buffer += cat_features_count; + } +} + +/// Calc hash for string cat feature at ps positions. +template +void calcStringHashes(const Column * column, size_t ps, const int ** buffer, const CatBoostLibraryHandler::APIHolder & api) +{ + size_t column_size = column->size(); + for (size_t j = 0; j < column_size; ++j) + { + auto ref = column->getDataAt(j); + const_cast(*buffer)[ps] = api.GetStringCatFeatureHash(ref.data, ref.size); + ++buffer; + } +} + +/// Calc hash for int cat feature at ps position. Buffer at positions ps should contains unhashed values. +void calcIntHashes(size_t column_size, size_t ps, const int ** buffer, const CatBoostLibraryHandler::APIHolder & api) +{ + for (size_t j = 0; j < column_size; ++j) + { + const_cast(*buffer)[ps] = api.GetIntegerCatFeatureHash((*buffer)[ps]); + ++buffer; + } +} + +/// buffer contains column->size() rows and size columns. +/// For int cat features calc hash inplace. +/// For string cat features calc hash from column rows. +void calcHashes(const ColumnRawPtrs & columns, size_t offset, size_t size, const int ** buffer, const CatBoostLibraryHandler::APIHolder & api) +{ + if (size == 0) + return; + size_t column_size = columns[offset]->size(); + + std::vector> data; + for (size_t i = 0; i < size; ++i) + { + const auto * column = columns[offset + i]; + if (const auto * column_string = typeid_cast(column)) + calcStringHashes(column_string, i, buffer, api); + else if (const auto * column_fixed_string = typeid_cast(column)) + calcStringHashes(column_fixed_string, i, buffer, api); + else + calcIntHashes(column_size, i, buffer, api); + } +} + +} + +/// Convert values to row-oriented format and call evaluation function from CatBoost wrapper api. +/// * CalcModelPredictionFlat if no cat features +/// * CalcModelPrediction if all cat features are strings +/// * CalcModelPredictionWithHashedCatFeatures if has int cat features. +ColumnFloat64::MutablePtr CatBoostLibraryHandler::evalImpl( + const ColumnRawPtrs & columns, + bool cat_features_are_strings) const +{ + std::string error_msg = "Error occurred while applying CatBoost model: "; + size_t column_size = columns.front()->size(); + + auto result = ColumnFloat64::create(column_size * tree_count); + auto * result_buf = result->getData().data(); + + if (!column_size) + return result; + + /// Prepare float features. + PODArray float_features(column_size); + auto * float_features_buf = float_features.data(); + /// Store all float data into single column. float_features is a list of pointers to it. + auto float_features_col = placeNumericColumns(columns, 0, float_features_count, float_features_buf); + + if (cat_features_count == 0) + { + if (!api.CalcModelPredictionFlat(model_calcer_handle, column_size, + float_features_buf, float_features_count, + result_buf, column_size * tree_count)) + { + + throw Exception(error_msg + api.GetErrorString(), ErrorCodes::CANNOT_APPLY_CATBOOST_MODEL); + } + return result; + } + + /// Prepare cat features. + if (cat_features_are_strings) + { + /// cat_features_holder stores pointers to ColumnString data or fixed_strings_data. + PODArray cat_features_holder(cat_features_count * column_size); + PODArray cat_features(column_size); + auto * cat_features_buf = cat_features.data(); + + fillCatFeaturesBuffer(cat_features_buf, cat_features_holder.data(), column_size, cat_features_count); + /// Fixed strings are stored without termination zero, so have to copy data into fixed_strings_data. + auto fixed_strings_data = placeStringColumns(columns, float_features_count, + cat_features_count, cat_features_holder.data()); + + if (!api.CalcModelPrediction(model_calcer_handle, column_size, + float_features_buf, float_features_count, + cat_features_buf, cat_features_count, + result_buf, column_size * tree_count)) + { + throw Exception(error_msg + api.GetErrorString(), ErrorCodes::CANNOT_APPLY_CATBOOST_MODEL); + } + } + else + { + PODArray cat_features(column_size); + auto * cat_features_buf = cat_features.data(); + auto cat_features_col = placeNumericColumns(columns, float_features_count, + cat_features_count, cat_features_buf); + calcHashes(columns, float_features_count, cat_features_count, cat_features_buf, api); + if (!api.CalcModelPredictionWithHashedCatFeatures( + model_calcer_handle, column_size, + float_features_buf, float_features_count, + cat_features_buf, cat_features_count, + result_buf, column_size * tree_count)) + { + throw Exception(error_msg + api.GetErrorString(), ErrorCodes::CANNOT_APPLY_CATBOOST_MODEL); + } + } + + return result; +} + +size_t CatBoostLibraryHandler::getTreeCount() const +{ + std::lock_guard lock(mutex); + return tree_count; +} + +ColumnPtr CatBoostLibraryHandler::evaluate(const ColumnRawPtrs & columns) const +{ + std::lock_guard lock(mutex); + + if (columns.empty()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Got empty columns list for CatBoost model."); + + if (columns.size() != float_features_count + cat_features_count) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Number of columns is different with number of features: columns size {} float features size {} + cat features size {}", + columns.size(), + float_features_count, + cat_features_count); + + for (size_t i = 0; i < float_features_count; ++i) + { + if (!columns[i]->isNumeric()) + { + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Column {} should be numeric to make float feature.", i); + } + } + + bool cat_features_are_strings = true; + for (size_t i = float_features_count; i < float_features_count + cat_features_count; ++i) + { + const auto * column = columns[i]; + if (column->isNumeric()) + { + cat_features_are_strings = false; + } + else if (!(typeid_cast(column) + || typeid_cast(column))) + { + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Column {} should be numeric or string.", i); + } + } + + auto result = evalImpl(columns, cat_features_are_strings); + + if (tree_count == 1) + return result; + + size_t column_size = columns.front()->size(); + auto * result_buf = result->getData().data(); + + /// Multiple trees case. Copy data to several columns. + MutableColumns mutable_columns(tree_count); + std::vector column_ptrs(tree_count); + for (size_t i = 0; i < tree_count; ++i) + { + auto col = ColumnFloat64::create(column_size); + column_ptrs[i] = col->getData().data(); + mutable_columns[i] = std::move(col); + } + + Float64 * data = result_buf; + for (size_t row = 0; row < column_size; ++row) + { + for (size_t i = 0; i < tree_count; ++i) + { + *column_ptrs[i] = *data; + ++column_ptrs[i]; + ++data; + } + } + + return ColumnTuple::create(std::move(mutable_columns)); +} + +} diff --git a/programs/library-bridge/CatBoostLibraryHandler.h b/programs/library-bridge/CatBoostLibraryHandler.h new file mode 100644 index 00000000000..a20e1412552 --- /dev/null +++ b/programs/library-bridge/CatBoostLibraryHandler.h @@ -0,0 +1,79 @@ +#pragma once + +#include "CatBoostLibraryAPI.h" + +#include +#include +#include +#include +#include +#include +#include "SharedLibrary.h" + +#include +#include + + +namespace DB +{ + +/// Abstracts access to the CatBoost shared library. +class CatBoostLibraryHandler +{ +public: + /// Holds pointers to CatBoost library functions + struct APIHolder + { + explicit APIHolder(SharedLibrary & lib); + + // NOLINTBEGIN(readability-identifier-naming) + CatBoostLibraryAPI::ModelCalcerCreateFunc ModelCalcerCreate; + CatBoostLibraryAPI::ModelCalcerDeleteFunc ModelCalcerDelete; + CatBoostLibraryAPI::GetErrorStringFunc GetErrorString; + CatBoostLibraryAPI::LoadFullModelFromFileFunc LoadFullModelFromFile; + CatBoostLibraryAPI::CalcModelPredictionFlatFunc CalcModelPredictionFlat; + CatBoostLibraryAPI::CalcModelPredictionFunc CalcModelPrediction; + CatBoostLibraryAPI::CalcModelPredictionWithHashedCatFeaturesFunc CalcModelPredictionWithHashedCatFeatures; + CatBoostLibraryAPI::GetStringCatFeatureHashFunc GetStringCatFeatureHash; + CatBoostLibraryAPI::GetIntegerCatFeatureHashFunc GetIntegerCatFeatureHash; + CatBoostLibraryAPI::GetFloatFeaturesCountFunc GetFloatFeaturesCount; + CatBoostLibraryAPI::GetCatFeaturesCountFunc GetCatFeaturesCount; + CatBoostLibraryAPI::GetTreeCountFunc GetTreeCount; + CatBoostLibraryAPI::GetDimensionsCountFunc GetDimensionsCount; + // NOLINTEND(readability-identifier-naming) + }; + + CatBoostLibraryHandler( + const String & library_path, + const String & model_path); + + ~CatBoostLibraryHandler(); + + std::chrono::system_clock::time_point getLoadingStartTime() const; + std::chrono::milliseconds getLoadingDuration() const; + + size_t getTreeCount() const; + + ColumnPtr evaluate(const ColumnRawPtrs & columns) const; + +private: + std::chrono::system_clock::time_point loading_start_time; + std::chrono::milliseconds loading_duration; + + const SharedLibraryPtr library; + const APIHolder api; + + mutable std::mutex mutex; + + CatBoostLibraryAPI::ModelCalcerHandle * model_calcer_handle TSA_GUARDED_BY(mutex) TSA_PT_GUARDED_BY(mutex); + + size_t float_features_count TSA_GUARDED_BY(mutex); + size_t cat_features_count TSA_GUARDED_BY(mutex); + size_t tree_count TSA_GUARDED_BY(mutex); + + ColumnFloat64::MutablePtr evalImpl(const ColumnRawPtrs & columns, bool cat_features_are_strings) const TSA_REQUIRES(mutex); +}; + +using CatBoostLibraryHandlerPtr = std::shared_ptr; + +} diff --git a/programs/library-bridge/CatBoostLibraryHandlerFactory.cpp b/programs/library-bridge/CatBoostLibraryHandlerFactory.cpp new file mode 100644 index 00000000000..6ee078f6c5c --- /dev/null +++ b/programs/library-bridge/CatBoostLibraryHandlerFactory.cpp @@ -0,0 +1,80 @@ +#include "CatBoostLibraryHandlerFactory.h" + +#include + + +namespace DB +{ + +CatBoostLibraryHandlerFactory & CatBoostLibraryHandlerFactory::instance() +{ + static CatBoostLibraryHandlerFactory instance; + return instance; +} + +CatBoostLibraryHandlerFactory::CatBoostLibraryHandlerFactory() + : log(&Poco::Logger::get("CatBoostLibraryHandlerFactory")) +{ +} + +CatBoostLibraryHandlerPtr CatBoostLibraryHandlerFactory::tryGetModel(const String & model_path, const String & library_path, bool create_if_not_found) +{ + std::lock_guard lock(mutex); + + auto handler = library_handlers.find(model_path); + bool found = (handler != library_handlers.end()); + + if (found) + return handler->second; + else + { + if (create_if_not_found) + { + auto new_handler = std::make_shared(library_path, model_path); + library_handlers.emplace(model_path, new_handler); + LOG_DEBUG(log, "Loaded catboost library handler for model path '{}'", model_path); + return new_handler; + } + return nullptr; + } +} + +void CatBoostLibraryHandlerFactory::removeModel(const String & model_path) +{ + std::lock_guard lock(mutex); + + bool deleted = library_handlers.erase(model_path); + if (!deleted) + { + LOG_DEBUG(log, "Cannot unload catboost library handler for model path '{}'", model_path); + return; + } + LOG_DEBUG(log, "Unloaded catboost library handler for model path '{}'", model_path); +} + +void CatBoostLibraryHandlerFactory::removeAllModels() +{ + std::lock_guard lock(mutex); + library_handlers.clear(); + LOG_DEBUG(log, "Unloaded all catboost library handlers"); +} + +ExternalModelInfos CatBoostLibraryHandlerFactory::getModelInfos() +{ + std::lock_guard lock(mutex); + + ExternalModelInfos result; + + for (const auto & handler : library_handlers) + result.push_back({ + .model_path = handler.first, + .model_type = "catboost", + .loading_start_time = handler.second->getLoadingStartTime(), + .loading_duration = handler.second->getLoadingDuration() + }); + + return result; + +} + +} diff --git a/programs/library-bridge/CatBoostLibraryHandlerFactory.h b/programs/library-bridge/CatBoostLibraryHandlerFactory.h new file mode 100644 index 00000000000..6ba3fe84ec9 --- /dev/null +++ b/programs/library-bridge/CatBoostLibraryHandlerFactory.h @@ -0,0 +1,37 @@ +#pragma once + +#include "CatBoostLibraryHandler.h" + +#include +#include + +#include +#include +#include + + +namespace DB +{ + +class CatBoostLibraryHandlerFactory final : private boost::noncopyable +{ +public: + static CatBoostLibraryHandlerFactory & instance(); + + CatBoostLibraryHandlerFactory(); + + CatBoostLibraryHandlerPtr tryGetModel(const String & model_path, const String & library_path, bool create_if_not_found); + + void removeModel(const String & model_path); + void removeAllModels(); + + ExternalModelInfos getModelInfos(); + +private: + /// map: model path --> catboost library handler + std::unordered_map library_handlers TSA_GUARDED_BY(mutex); + std::mutex mutex; + Poco::Logger * log; +}; + +} diff --git a/programs/library-bridge/ExternalDictionaryLibraryHandler.h b/programs/library-bridge/ExternalDictionaryLibraryHandler.h index 7713e9a6830..77c9b9fdf39 100644 --- a/programs/library-bridge/ExternalDictionaryLibraryHandler.h +++ b/programs/library-bridge/ExternalDictionaryLibraryHandler.h @@ -1,6 +1,6 @@ #pragma once -#include +#include "SharedLibrary.h" #include #include "ExternalDictionaryLibraryUtils.h" @@ -50,6 +50,6 @@ private: void * lib_data; }; -using SharedLibraryHandlerPtr = std::shared_ptr; +using ExternalDictionaryLibraryHandlerPtr = std::shared_ptr; } diff --git a/programs/library-bridge/ExternalDictionaryLibraryHandlerFactory.cpp b/programs/library-bridge/ExternalDictionaryLibraryHandlerFactory.cpp index ffa5ff6f493..6acd9af20ed 100644 --- a/programs/library-bridge/ExternalDictionaryLibraryHandlerFactory.cpp +++ b/programs/library-bridge/ExternalDictionaryLibraryHandlerFactory.cpp @@ -1,37 +1,40 @@ #include "ExternalDictionaryLibraryHandlerFactory.h" +#include namespace DB { -SharedLibraryHandlerPtr ExternalDictionaryLibraryHandlerFactory::get(const std::string & dictionary_id) +ExternalDictionaryLibraryHandlerPtr ExternalDictionaryLibraryHandlerFactory::get(const String & dictionary_id) { std::lock_guard lock(mutex); - auto library_handler = library_handlers.find(dictionary_id); - - if (library_handler != library_handlers.end()) - return library_handler->second; + if (auto handler = library_handlers.find(dictionary_id); handler != library_handlers.end()) + return handler->second; return nullptr; } void ExternalDictionaryLibraryHandlerFactory::create( - const std::string & dictionary_id, - const std::string & library_path, - const std::vector & library_settings, + const String & dictionary_id, + const String & library_path, + const std::vector & library_settings, const Block & sample_block, - const std::vector & attributes_names) + const std::vector & attributes_names) { std::lock_guard lock(mutex); - if (!library_handlers.contains(dictionary_id)) - library_handlers.emplace(std::make_pair(dictionary_id, std::make_shared(library_path, library_settings, sample_block, attributes_names))); - else + + if (library_handlers.contains(dictionary_id)) + { LOG_WARNING(&Poco::Logger::get("ExternalDictionaryLibraryHandlerFactory"), "Library handler with dictionary id {} already exists", dictionary_id); + return; + } + + library_handlers.emplace(std::make_pair(dictionary_id, std::make_shared(library_path, library_settings, sample_block, attributes_names))); } -bool ExternalDictionaryLibraryHandlerFactory::clone(const std::string & from_dictionary_id, const std::string & to_dictionary_id) +bool ExternalDictionaryLibraryHandlerFactory::clone(const String & from_dictionary_id, const String & to_dictionary_id) { std::lock_guard lock(mutex); auto from_library_handler = library_handlers.find(from_dictionary_id); @@ -45,7 +48,7 @@ bool ExternalDictionaryLibraryHandlerFactory::clone(const std::string & from_dic } -bool ExternalDictionaryLibraryHandlerFactory::remove(const std::string & dictionary_id) +bool ExternalDictionaryLibraryHandlerFactory::remove(const String & dictionary_id) { std::lock_guard lock(mutex); /// extDict_libDelete is called in destructor. diff --git a/programs/library-bridge/ExternalDictionaryLibraryHandlerFactory.h b/programs/library-bridge/ExternalDictionaryLibraryHandlerFactory.h index d821270c474..3dfafd82a0f 100644 --- a/programs/library-bridge/ExternalDictionaryLibraryHandlerFactory.h +++ b/programs/library-bridge/ExternalDictionaryLibraryHandlerFactory.h @@ -17,22 +17,22 @@ class ExternalDictionaryLibraryHandlerFactory final : private boost::noncopyable public: static ExternalDictionaryLibraryHandlerFactory & instance(); - SharedLibraryHandlerPtr get(const std::string & dictionary_id); + ExternalDictionaryLibraryHandlerPtr get(const String & dictionary_id); void create( - const std::string & dictionary_id, - const std::string & library_path, - const std::vector & library_settings, + const String & dictionary_id, + const String & library_path, + const std::vector & library_settings, const Block & sample_block, - const std::vector & attributes_names); + const std::vector & attributes_names); - bool clone(const std::string & from_dictionary_id, const std::string & to_dictionary_id); + bool clone(const String & from_dictionary_id, const String & to_dictionary_id); - bool remove(const std::string & dictionary_id); + bool remove(const String & dictionary_id); private: /// map: dict_id -> sharedLibraryHandler - std::unordered_map library_handlers TSA_GUARDED_BY(mutex); + std::unordered_map library_handlers TSA_GUARDED_BY(mutex); std::mutex mutex; }; diff --git a/programs/library-bridge/LibraryBridgeHandlerFactory.cpp b/programs/library-bridge/LibraryBridgeHandlerFactory.cpp index f8f6a23e1be..4af1f8355e8 100644 --- a/programs/library-bridge/LibraryBridgeHandlerFactory.cpp +++ b/programs/library-bridge/LibraryBridgeHandlerFactory.cpp @@ -27,12 +27,16 @@ std::unique_ptr LibraryBridgeHandlerFactory::createRequestHa { if (uri.getPath() == "/extdict_ping") return std::make_unique(keep_alive_timeout, getContext()); + else if (uri.getPath() == "/catboost_ping") + return std::make_unique(keep_alive_timeout, getContext()); } if (request.getMethod() == Poco::Net::HTTPRequest::HTTP_POST) { if (uri.getPath() == "/extdict_request") return std::make_unique(keep_alive_timeout, getContext()); + else if (uri.getPath() == "/catboost_request") + return std::make_unique(keep_alive_timeout, getContext()); } return nullptr; diff --git a/programs/library-bridge/LibraryBridgeHandlers.cpp b/programs/library-bridge/LibraryBridgeHandlers.cpp index a28148bd1f7..ab81472be88 100644 --- a/programs/library-bridge/LibraryBridgeHandlers.cpp +++ b/programs/library-bridge/LibraryBridgeHandlers.cpp @@ -1,24 +1,32 @@ #include "LibraryBridgeHandlers.h" + +#include "CatBoostLibraryHandler.h" +#include "CatBoostLibraryHandlerFactory.h" +#include "ExternalDictionaryLibraryHandler.h" #include "ExternalDictionaryLibraryHandlerFactory.h" #include -#include -#include +#include #include #include +#include +#include #include #include -#include #include -#include -#include -#include #include #include +#include +#include #include #include +#include #include -#include +#include +#include +#include +#include +#include namespace DB @@ -31,7 +39,7 @@ namespace ErrorCodes namespace { - void processError(HTTPServerResponse & response, const std::string & message) + void processError(HTTPServerResponse & response, const String & message) { response.setStatusAndReason(HTTPResponse::HTTP_INTERNAL_SERVER_ERROR); @@ -41,7 +49,7 @@ namespace LOG_WARNING(&Poco::Logger::get("LibraryBridge"), fmt::runtime(message)); } - std::shared_ptr parseColumns(std::string && column_string) + std::shared_ptr parseColumns(String && column_string) { auto sample_block = std::make_shared(); auto names_and_types = NamesAndTypesList::parse(column_string); @@ -59,10 +67,10 @@ namespace return ids; } - std::vector parseNamesFromBinary(const std::string & names_string) + std::vector parseNamesFromBinary(const String & names_string) { ReadBufferFromString buf(names_string); - std::vector names; + std::vector names; readVectorBinary(names, buf); return names; } @@ -79,13 +87,15 @@ static void writeData(Block data, OutputFormatPtr format) executor.execute(); } + ExternalDictionaryLibraryBridgeRequestHandler::ExternalDictionaryLibraryBridgeRequestHandler(size_t keep_alive_timeout_, ContextPtr context_) : WithContext(context_) - , log(&Poco::Logger::get("ExternalDictionaryLibraryBridgeRequestHandler")) , keep_alive_timeout(keep_alive_timeout_) + , log(&Poco::Logger::get("ExternalDictionaryLibraryBridgeRequestHandler")) { } + void ExternalDictionaryLibraryBridgeRequestHandler::handleRequest(HTTPServerRequest & request, HTTPServerResponse & response) { LOG_TRACE(log, "Request URI: {}", request.getURI()); @@ -97,7 +107,7 @@ void ExternalDictionaryLibraryBridgeRequestHandler::handleRequest(HTTPServerRequ version = 0; /// assumed version for too old servers which do not send a version else { - String version_str = params.get("version"); + const String & version_str = params.get("version"); if (!tryParse(version, version_str)) { processError(response, "Unable to parse 'version' string in request URL: '" + version_str + "' Check if the server and library-bridge have the same version."); @@ -124,8 +134,8 @@ void ExternalDictionaryLibraryBridgeRequestHandler::handleRequest(HTTPServerRequ return; } - std::string method = params.get("method"); - std::string dictionary_id = params.get("dictionary_id"); + const String & method = params.get("method"); + const String & dictionary_id = params.get("dictionary_id"); LOG_TRACE(log, "Library method: '{}', dictionary id: {}", method, dictionary_id); WriteBufferFromHTTPServerResponse out(response, request.getMethod() == Poco::Net::HTTPRequest::HTTP_HEAD, keep_alive_timeout); @@ -141,7 +151,7 @@ void ExternalDictionaryLibraryBridgeRequestHandler::handleRequest(HTTPServerRequ return; } - std::string from_dictionary_id = params.get("from_dictionary_id"); + const String & from_dictionary_id = params.get("from_dictionary_id"); bool cloned = false; cloned = ExternalDictionaryLibraryHandlerFactory::instance().clone(from_dictionary_id, dictionary_id); @@ -166,7 +176,7 @@ void ExternalDictionaryLibraryBridgeRequestHandler::handleRequest(HTTPServerRequ return; } - std::string library_path = params.get("library_path"); + const String & library_path = params.get("library_path"); if (!params.has("library_settings")) { @@ -174,10 +184,10 @@ void ExternalDictionaryLibraryBridgeRequestHandler::handleRequest(HTTPServerRequ return; } - const auto & settings_string = params.get("library_settings"); + const String & settings_string = params.get("library_settings"); LOG_DEBUG(log, "Parsing library settings from binary string"); - std::vector library_settings = parseNamesFromBinary(settings_string); + std::vector library_settings = parseNamesFromBinary(settings_string); /// Needed for library dictionary if (!params.has("attributes_names")) @@ -186,10 +196,10 @@ void ExternalDictionaryLibraryBridgeRequestHandler::handleRequest(HTTPServerRequ return; } - const auto & attributes_string = params.get("attributes_names"); + const String & attributes_string = params.get("attributes_names"); LOG_DEBUG(log, "Parsing attributes names from binary string"); - std::vector attributes_names = parseNamesFromBinary(attributes_string); + std::vector attributes_names = parseNamesFromBinary(attributes_string); /// Needed to parse block from binary string format if (!params.has("sample_block")) @@ -197,7 +207,7 @@ void ExternalDictionaryLibraryBridgeRequestHandler::handleRequest(HTTPServerRequ processError(response, "No 'sample_block' in request URL"); return; } - std::string sample_block_string = params.get("sample_block"); + String sample_block_string = params.get("sample_block"); std::shared_ptr sample_block; try @@ -297,7 +307,7 @@ void ExternalDictionaryLibraryBridgeRequestHandler::handleRequest(HTTPServerRequ return; } - std::string requested_block_string = params.get("requested_block_sample"); + String requested_block_string = params.get("requested_block_sample"); std::shared_ptr requested_sample_block; try @@ -332,7 +342,8 @@ void ExternalDictionaryLibraryBridgeRequestHandler::handleRequest(HTTPServerRequ } else { - LOG_WARNING(log, "Unknown library method: '{}'", method); + processError(response, "Unknown library method '" + method + "'"); + LOG_ERROR(log, "Unknown library method: '{}'", method); } } catch (...) @@ -362,6 +373,7 @@ void ExternalDictionaryLibraryBridgeRequestHandler::handleRequest(HTTPServerRequ } } + ExternalDictionaryLibraryBridgeExistsHandler::ExternalDictionaryLibraryBridgeExistsHandler(size_t keep_alive_timeout_, ContextPtr context_) : WithContext(context_) , keep_alive_timeout(keep_alive_timeout_) @@ -369,6 +381,7 @@ ExternalDictionaryLibraryBridgeExistsHandler::ExternalDictionaryLibraryBridgeExi { } + void ExternalDictionaryLibraryBridgeExistsHandler::handleRequest(HTTPServerRequest & request, HTTPServerResponse & response) { try @@ -382,7 +395,7 @@ void ExternalDictionaryLibraryBridgeExistsHandler::handleRequest(HTTPServerReque return; } - std::string dictionary_id = params.get("dictionary_id"); + const String & dictionary_id = params.get("dictionary_id"); auto library_handler = ExternalDictionaryLibraryHandlerFactory::instance().get(dictionary_id); @@ -399,4 +412,230 @@ void ExternalDictionaryLibraryBridgeExistsHandler::handleRequest(HTTPServerReque } +CatBoostLibraryBridgeRequestHandler::CatBoostLibraryBridgeRequestHandler( + size_t keep_alive_timeout_, ContextPtr context_) + : WithContext(context_) + , keep_alive_timeout(keep_alive_timeout_) + , log(&Poco::Logger::get("CatBoostLibraryBridgeRequestHandler")) +{ +} + + +void CatBoostLibraryBridgeRequestHandler::handleRequest(HTTPServerRequest & request, HTTPServerResponse & response) +{ + LOG_TRACE(log, "Request URI: {}", request.getURI()); + HTMLForm params(getContext()->getSettingsRef(), request); + + size_t version; + + if (!params.has("version")) + version = 0; /// assumed version for too old servers which do not send a version + else + { + const String & version_str = params.get("version"); + if (!tryParse(version, version_str)) + { + processError(response, "Unable to parse 'version' string in request URL: '" + version_str + "' Check if the server and library-bridge have the same version."); + return; + } + } + + if (version != LIBRARY_BRIDGE_PROTOCOL_VERSION) + { + /// backwards compatibility is considered unnecessary for now, just let the user know that the server and the bridge must be upgraded together + processError(response, "Server and library-bridge have different versions: '" + std::to_string(version) + "' vs. '" + std::to_string(LIBRARY_BRIDGE_PROTOCOL_VERSION) + "'"); + return; + } + if (!params.has("method")) + { + processError(response, "No 'method' in request URL"); + return; + } + + const String & method = params.get("method"); + + LOG_TRACE(log, "Library method: '{}'", method); + WriteBufferFromHTTPServerResponse out(response, request.getMethod() == Poco::Net::HTTPRequest::HTTP_HEAD, keep_alive_timeout); + + try + { + if (method == "catboost_list") + { + ExternalModelInfos model_infos = CatBoostLibraryHandlerFactory::instance().getModelInfos(); + + writeIntBinary(static_cast(model_infos.size()), out); + + for (const auto & info : model_infos) + { + writeStringBinary(info.model_path, out); + writeStringBinary(info.model_type, out); + + UInt64 t = std::chrono::system_clock::to_time_t(info.loading_start_time); + writeIntBinary(t, out); + + t = info.loading_duration.count(); + writeIntBinary(t, out); + + } + } + else if (method == "catboost_removeModel") + { + auto & read_buf = request.getStream(); + params.read(read_buf); + + if (!params.has("model_path")) + { + processError(response, "No 'model_path' in request URL"); + return; + } + + const String & model_path = params.get("model_path"); + + CatBoostLibraryHandlerFactory::instance().removeModel(model_path); + + String res = "1"; + writeStringBinary(res, out); + } + else if (method == "catboost_removeAllModels") + { + CatBoostLibraryHandlerFactory::instance().removeAllModels(); + + String res = "1"; + writeStringBinary(res, out); + } + else if (method == "catboost_GetTreeCount") + { + auto & read_buf = request.getStream(); + params.read(read_buf); + + if (!params.has("library_path")) + { + processError(response, "No 'library_path' in request URL"); + return; + } + + const String & library_path = params.get("library_path"); + + if (!params.has("model_path")) + { + processError(response, "No 'model_path' in request URL"); + return; + } + + const String & model_path = params.get("model_path"); + + auto catboost_handler = CatBoostLibraryHandlerFactory::instance().tryGetModel(model_path, library_path, /*create_if_not_found*/ true); + size_t tree_count = catboost_handler->getTreeCount(); + writeIntBinary(tree_count, out); + } + else if (method == "catboost_libEvaluate") + { + auto & read_buf = request.getStream(); + params.read(read_buf); + + if (!params.has("model_path")) + { + processError(response, "No 'model_path' in request URL"); + return; + } + + const String & model_path = params.get("model_path"); + + if (!params.has("data")) + { + processError(response, "No 'data' in request URL"); + return; + } + + const String & data = params.get("data"); + + ReadBufferFromString string_read_buf(data); + NativeReader deserializer(string_read_buf, /*server_revision*/ 0); + Block block_read = deserializer.read(); + + Columns col_ptrs = block_read.getColumns(); + ColumnRawPtrs col_raw_ptrs; + for (const auto & p : col_ptrs) + col_raw_ptrs.push_back(&*p); + + auto catboost_handler = CatBoostLibraryHandlerFactory::instance().tryGetModel(model_path, "DummyLibraryPath", /*create_if_not_found*/ false); + + if (!catboost_handler) + { + processError(response, "CatBoost library is not loaded for model '" + model_path + "'. Please try again."); + return; + } + + ColumnPtr res_col = catboost_handler->evaluate(col_raw_ptrs); + + DataTypePtr res_col_type = std::make_shared(); + String res_col_name = "res_col"; + + ColumnsWithTypeAndName res_cols_with_type_and_name = {{res_col, res_col_type, res_col_name}}; + + Block block_write(res_cols_with_type_and_name); + NativeWriter serializer{out, /*client_revision*/ 0, block_write}; + serializer.write(block_write); + } + else + { + processError(response, "Unknown library method '" + method + "'"); + LOG_ERROR(log, "Unknown library method: '{}'", method); + } + } + catch (...) + { + auto message = getCurrentExceptionMessage(true); + LOG_ERROR(log, "Failed to process request. Error: {}", message); + + response.setStatusAndReason(Poco::Net::HTTPResponse::HTTP_INTERNAL_SERVER_ERROR, message); // can't call process_error, because of too soon response sending + try + { + writeStringBinary(message, out); + out.finalize(); + } + catch (...) + { + tryLogCurrentException(log); + } + } + + try + { + out.finalize(); + } + catch (...) + { + tryLogCurrentException(log); + } +} + + +CatBoostLibraryBridgeExistsHandler::CatBoostLibraryBridgeExistsHandler(size_t keep_alive_timeout_, ContextPtr context_) + : WithContext(context_) + , keep_alive_timeout(keep_alive_timeout_) + , log(&Poco::Logger::get("CatBoostLibraryBridgeExistsHandler")) +{ +} + + +void CatBoostLibraryBridgeExistsHandler::handleRequest(HTTPServerRequest & request, HTTPServerResponse & response) +{ + try + { + LOG_TRACE(log, "Request URI: {}", request.getURI()); + HTMLForm params(getContext()->getSettingsRef(), request); + + String res = "1"; + + setResponseDefaultHeaders(response, keep_alive_timeout); + LOG_TRACE(log, "Sending ping response: {}", res); + response.sendBuffer(res.data(), res.size()); + } + catch (...) + { + tryLogCurrentException("PingHandler"); + } +} + } diff --git a/programs/library-bridge/LibraryBridgeHandlers.h b/programs/library-bridge/LibraryBridgeHandlers.h index b20f40616ce..16815e84723 100644 --- a/programs/library-bridge/LibraryBridgeHandlers.h +++ b/programs/library-bridge/LibraryBridgeHandlers.h @@ -1,9 +1,8 @@ #pragma once +#include #include #include -#include -#include "ExternalDictionaryLibraryHandler.h" namespace DB @@ -26,11 +25,12 @@ public: private: static constexpr inline auto FORMAT = "RowBinary"; + const size_t keep_alive_timeout; Poco::Logger * log; - size_t keep_alive_timeout; }; +// Handler for checking if the external dictionary library is loaded (used for handshake) class ExternalDictionaryLibraryBridgeExistsHandler : public HTTPRequestHandler, WithContext { public: @@ -43,4 +43,47 @@ private: Poco::Logger * log; }; + +/// Handler for requests to catboost library. The call protocol is as follows: +/// (1) Send a "catboost_GetTreeCount" request from the server to the bridge. It contains a library path (e.g /home/user/libcatboost.so) and +/// a model path (e.g. /home/user/model.bin). This loads the catboost library handler associated with the model path, then executes +/// GetTreeCount() on the library handler and sends the result back to the server. +/// (2) Send "catboost_Evaluate" from the server to the bridge. It contains a model path and the features to run the interference on. Step +/// (2) is called multiple times (once per chunk) by the server. +/// +/// We would ideally like to have steps (1) and (2) in one atomic handler but can't because the evaluation on the server side is divided +/// into two dependent phases: FunctionCatBoostEvaluate::getReturnTypeImpl() and ::executeImpl(). So the model may in principle be unloaded +/// from the library-bridge between steps (1) and (2). Step (2) checks if that is the case and fails gracefully. This is okay because that +/// situation considered exceptional and rare. +/// +/// An update of a model is performed by unloading it. The first call to "catboost_GetTreeCount" brings it into memory again. +/// +/// Further handlers are provided for unloading a specific model, for unloading all models or for retrieving information about the loaded +/// models for display in a system view. +class CatBoostLibraryBridgeRequestHandler : public HTTPRequestHandler, WithContext +{ +public: + CatBoostLibraryBridgeRequestHandler(size_t keep_alive_timeout_, ContextPtr context_); + + void handleRequest(HTTPServerRequest & request, HTTPServerResponse & response) override; + +private: + const size_t keep_alive_timeout; + Poco::Logger * log; +}; + + +// Handler for pinging the library-bridge for catboost access (used for handshake) +class CatBoostLibraryBridgeExistsHandler : public HTTPRequestHandler, WithContext +{ +public: + CatBoostLibraryBridgeExistsHandler(size_t keep_alive_timeout_, ContextPtr context_); + + void handleRequest(HTTPServerRequest & request, HTTPServerResponse & response) override; + +private: + const size_t keep_alive_timeout; + Poco::Logger * log; +}; + } diff --git a/src/Common/SharedLibrary.cpp b/programs/library-bridge/SharedLibrary.cpp similarity index 95% rename from src/Common/SharedLibrary.cpp rename to programs/library-bridge/SharedLibrary.cpp index 6104c96676a..d70709474b5 100644 --- a/src/Common/SharedLibrary.cpp +++ b/programs/library-bridge/SharedLibrary.cpp @@ -1,8 +1,7 @@ #include "SharedLibrary.h" #include -#include #include -#include "Exception.h" +#include namespace DB diff --git a/src/Common/SharedLibrary.h b/programs/library-bridge/SharedLibrary.h similarity index 100% rename from src/Common/SharedLibrary.h rename to programs/library-bridge/SharedLibrary.h diff --git a/programs/local/clickhouse-local.cpp b/programs/local/clickhouse-local.cpp index 5cc98ab6067..4b17e89496d 100644 --- a/programs/local/clickhouse-local.cpp +++ b/programs/local/clickhouse-local.cpp @@ -1,2 +1 @@ int mainEntryClickHouseLocal(int argc, char ** argv); -int main(int argc_, char ** argv_) { return mainEntryClickHouseLocal(argc_, argv_); } diff --git a/programs/main.cpp b/programs/main.cpp index fef0ad688e2..9b0e890cd76 100644 --- a/programs/main.cpp +++ b/programs/main.cpp @@ -402,6 +402,36 @@ void checkHarmfulEnvironmentVariables(char ** argv) } +/// Don't allow dlopen in the main ClickHouse binary, because it is harmful and insecure. +/// We don't use it. But it can be used by some libraries for implementation of "plugins". +/// We absolutely discourage the ancient technique of loading +/// 3rd-party uncontrolled dangerous libraries into the process address space, +/// because it is insane. + +extern "C" +{ + void * dlopen(const char *, int) + { + return nullptr; + } + + void * dlmopen(long, const char *, int) // NOLINT + { + return nullptr; + } + + int dlclose(void *) + { + return 0; + } + + const char * dlerror() + { + return "ClickHouse does not allow dynamic library loading"; + } +} + + /// This allows to implement assert to forbid initialization of a class in static constructors. /// Usage: /// @@ -422,6 +452,7 @@ int main(int argc_, char ** argv_) /// PHDR cache is required for query profiler to work reliably /// It also speed up exception handling, but exceptions from dynamically loaded libraries (dlopen) /// will work only after additional call of this function. + /// Note: we forbid dlopen in our code. updatePHDRCache(); #ifndef DISABLE_HARMFUL_ENV_VAR_CHECK diff --git a/programs/obfuscator/clickhouse-obfuscator.cpp b/programs/obfuscator/clickhouse-obfuscator.cpp index e57fa6d1b54..1f494223150 100644 --- a/programs/obfuscator/clickhouse-obfuscator.cpp +++ b/programs/obfuscator/clickhouse-obfuscator.cpp @@ -1,3 +1 @@ int mainEntryClickHouseObfuscator(int argc, char ** argv); -int main(int argc_, char ** argv_) { return mainEntryClickHouseObfuscator(argc_, argv_); } - diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 414766ee42a..5c09ba5b52e 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -51,7 +51,6 @@ #include #include #include -#include #include #include #include @@ -551,8 +550,9 @@ static void sanityChecks(Server & server) try { const char * filename = "/sys/devices/system/clocksource/clocksource0/current_clocksource"; - if (readString(filename).find("tsc") == std::string::npos) - server.context()->addWarningMessage("Linux is not using a fast TSC clock source. Performance can be degraded. Check " + String(filename)); + String clocksource = readString(filename); + if (clocksource.find("tsc") == std::string::npos && clocksource.find("kvm-clock") == std::string::npos) + server.context()->addWarningMessage("Linux is not using a fast clock source. Performance can be degraded. Check " + String(filename)); } catch (...) { @@ -1157,7 +1157,6 @@ int Server::main(const std::vector & /*args*/) global_context->setExternalAuthenticatorsConfig(*config); global_context->loadOrReloadDictionaries(*config); - global_context->loadOrReloadModels(*config); global_context->loadOrReloadUserDefinedExecutableFunctions(*config); global_context->setRemoteHostFilter(*config); @@ -1738,17 +1737,6 @@ int Server::main(const std::vector & /*args*/) throw; } - /// try to load models immediately, throw on error and die - try - { - global_context->loadOrReloadModels(config()); - } - catch (...) - { - tryLogCurrentException(log, "Caught exception while loading dictionaries."); - throw; - } - /// try to load user defined executable functions, throw on error and die try { diff --git a/programs/server/clickhouse-server.cpp b/programs/server/clickhouse-server.cpp index f49dc21d9fe..a7a3d8f00e3 100644 --- a/programs/server/clickhouse-server.cpp +++ b/programs/server/clickhouse-server.cpp @@ -1,24 +1 @@ -#include - -#include - - int mainEntryClickHouseServer(int argc, char ** argv); - -/** - * This is the entry-point for the split build server. The initialization - * is copied from single-binary entry point in main.cpp. - */ -int main(int argc_, char ** argv_) -{ - /// Reset new handler to default (that throws std::bad_alloc) - /// It is needed because LLVM library clobbers it. - std::set_new_handler(nullptr); - - /// PHDR cache is required for query profiler to work reliably - /// It also speed up exception handling, but exceptions from dynamically loaded libraries (dlopen) - /// will work only after additional call of this function. - updatePHDRCache(); - - return mainEntryClickHouseServer(argc_, argv_); -} diff --git a/programs/static-files-disk-uploader/clickhouse-static-files-disk-uploader.cpp b/programs/static-files-disk-uploader/clickhouse-static-files-disk-uploader.cpp index 063604b10b1..cf271f16a76 100644 --- a/programs/static-files-disk-uploader/clickhouse-static-files-disk-uploader.cpp +++ b/programs/static-files-disk-uploader/clickhouse-static-files-disk-uploader.cpp @@ -1,2 +1 @@ int mainEntryClickHouseStaticFilesDiskUploader(int argc, char ** argv); -int main(int argc_, char ** argv_) { return mainEntryClickHouseStaticFilesDiskUploader(argc_, argv_); } diff --git a/programs/su/clickhouse-su.cpp b/programs/su/clickhouse-su.cpp index bb0967ca271..9459ccad0e8 100644 --- a/programs/su/clickhouse-su.cpp +++ b/programs/su/clickhouse-su.cpp @@ -1,2 +1 @@ int mainEntryClickHouseSU(int argc, char ** argv); -int main(int argc_, char ** argv_) { return mainEntryClickHouseSU(argc_, argv_); } diff --git a/src/AggregateFunctions/AggregateFunctionCategoricalInformationValue.cpp b/src/AggregateFunctions/AggregateFunctionCategoricalInformationValue.cpp index 35654c08659..89ffdfa6109 100644 --- a/src/AggregateFunctions/AggregateFunctionCategoricalInformationValue.cpp +++ b/src/AggregateFunctions/AggregateFunctionCategoricalInformationValue.cpp @@ -1,12 +1,18 @@ -#include - +#include #include #include #include +#include +#include +#include +#include +#include +#include namespace DB { + struct Settings; namespace ErrorCodes @@ -15,6 +21,136 @@ namespace ErrorCodes extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; } +/** The function takes arguments x1, x2, ... xn, y. All arguments are bool. + * x arguments represents the fact that some category is true. + * + * It calculates how many times y was true and how many times y was false when every n-th category was true + * and the total number of times y was true and false. + * + * So, the size of the state is (n + 1) * 2 cells. + */ +class AggregateFunctionCategoricalIV final : public IAggregateFunctionHelper +{ +private: + using Counter = UInt64; + size_t category_count; + + static Counter & counter(AggregateDataPtr __restrict place, size_t i, bool what) + { + return reinterpret_cast(place)[i * 2 + (what ? 1 : 0)]; + } + + static const Counter & counter(ConstAggregateDataPtr __restrict place, size_t i, bool what) + { + return reinterpret_cast(place)[i * 2 + (what ? 1 : 0)]; + } + +public: + AggregateFunctionCategoricalIV(const DataTypes & arguments_, const Array & params_) : + IAggregateFunctionHelper{arguments_, params_}, + category_count{arguments_.size() - 1} + { + // notice: argument types has been checked before + } + + String getName() const override + { + return "categoricalInformationValue"; + } + + bool allocatesMemoryInArena() const override { return false; } + + void create(AggregateDataPtr __restrict place) const override + { + memset(place, 0, sizeOfData()); + } + + void destroy(AggregateDataPtr __restrict) const noexcept override + { + // nothing + } + + bool hasTrivialDestructor() const override + { + return true; + } + + size_t sizeOfData() const override + { + return sizeof(Counter) * (category_count + 1) * 2; + } + + size_t alignOfData() const override + { + return alignof(Counter); + } + + void add(AggregateDataPtr __restrict place, const IColumn ** columns, size_t row_num, Arena *) const override + { + const auto * y_col = static_cast(columns[category_count]); + bool y = y_col->getData()[row_num]; + + for (size_t i = 0; i < category_count; ++i) + { + const auto * x_col = static_cast(columns[i]); + bool x = x_col->getData()[row_num]; + + if (x) + ++counter(place, i, y); + } + + ++counter(place, category_count, y); + } + + void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, Arena *) const override + { + for (size_t i = 0; i <= category_count; ++i) + { + counter(place, i, false) += counter(rhs, i, false); + counter(place, i, true) += counter(rhs, i, true); + } + } + + void serialize(ConstAggregateDataPtr __restrict place, WriteBuffer & buf, std::optional /* version */) const override + { + buf.write(place, sizeOfData()); + } + + void deserialize(AggregateDataPtr __restrict place, ReadBuffer & buf, std::optional /* version */, Arena *) const override + { + buf.read(place, sizeOfData()); + } + + DataTypePtr getReturnType() const override + { + return std::make_shared( + std::make_shared>()); + } + + void insertResultInto(AggregateDataPtr __restrict place, IColumn & to, Arena *) const override /// NOLINT + { + auto & col = static_cast(to); + auto & data_col = static_cast(col.getData()); + auto & offset_col = static_cast(col.getOffsetsColumn()); + + data_col.reserve(data_col.size() + category_count); + + Float64 sum_no = static_cast(counter(place, category_count, false)); + Float64 sum_yes = static_cast(counter(place, category_count, true)); + + for (size_t i = 0; i < category_count; ++i) + { + Float64 no = static_cast(counter(place, i, false)); + Float64 yes = static_cast(counter(place, i, true)); + + data_col.insertValue((no / sum_no - yes / sum_yes) * (log((no / sum_no) / (yes / sum_yes)))); + } + + offset_col.insertValue(data_col.size()); + } +}; + + namespace { @@ -39,16 +175,15 @@ AggregateFunctionPtr createAggregateFunctionCategoricalIV( ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); } - return std::make_shared>(arguments, params); + return std::make_shared(arguments, params); } } -void registerAggregateFunctionCategoricalIV( - AggregateFunctionFactory & factory -) +void registerAggregateFunctionCategoricalIV(AggregateFunctionFactory & factory) { - factory.registerFunction("categoricalInformationValue", createAggregateFunctionCategoricalIV); + AggregateFunctionProperties properties = { .returns_default_when_only_null = true }; + factory.registerFunction("categoricalInformationValue", { createAggregateFunctionCategoricalIV, properties }); } } diff --git a/src/AggregateFunctions/AggregateFunctionCategoricalInformationValue.h b/src/AggregateFunctions/AggregateFunctionCategoricalInformationValue.h deleted file mode 100644 index 0e0db27cf22..00000000000 --- a/src/AggregateFunctions/AggregateFunctionCategoricalInformationValue.h +++ /dev/null @@ -1,135 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include -#include -#include - -#include - - -namespace DB -{ -struct Settings; - -template -class AggregateFunctionCategoricalIV final : public IAggregateFunctionHelper> -{ -private: - size_t category_count; - -public: - AggregateFunctionCategoricalIV(const DataTypes & arguments_, const Array & params_) : - IAggregateFunctionHelper> {arguments_, params_}, - category_count {arguments_.size() - 1} - { - // notice: argument types has been checked before - } - - String getName() const override - { - return "categoricalInformationValue"; - } - - bool allocatesMemoryInArena() const override { return false; } - - void create(AggregateDataPtr __restrict place) const override - { - memset(place, 0, sizeOfData()); - } - - void destroy(AggregateDataPtr __restrict) const noexcept override - { - // nothing - } - - bool hasTrivialDestructor() const override - { - return true; - } - - size_t sizeOfData() const override - { - return sizeof(T) * (category_count + 1) * 2; - } - - size_t alignOfData() const override - { - return alignof(T); - } - - void add(AggregateDataPtr __restrict place, const IColumn ** columns, size_t row_num, Arena *) const override - { - const auto * y_col = static_cast(columns[category_count]); - bool y = y_col->getData()[row_num]; - - for (size_t i : collections::range(0, category_count)) - { - const auto * x_col = static_cast(columns[i]); - bool x = x_col->getData()[row_num]; - - if (x) - reinterpret_cast(place)[i * 2 + size_t(y)] += 1; - } - - reinterpret_cast(place)[category_count * 2 + size_t(y)] += 1; - } - - void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, Arena *) const override - { - for (size_t i : collections::range(0, category_count + 1)) - { - reinterpret_cast(place)[i * 2] += reinterpret_cast(rhs)[i * 2]; - reinterpret_cast(place)[i * 2 + 1] += reinterpret_cast(rhs)[i * 2 + 1]; - } - } - - void serialize(ConstAggregateDataPtr __restrict place, WriteBuffer & buf, std::optional /* version */) const override - { - buf.write(place, sizeOfData()); - } - - void deserialize(AggregateDataPtr __restrict place, ReadBuffer & buf, std::optional /* version */, Arena *) const override - { - buf.read(place, sizeOfData()); - } - - DataTypePtr getReturnType() const override - { - return std::make_shared( - std::make_shared>() - ); - } - - void insertResultInto(AggregateDataPtr __restrict place, IColumn & to, Arena *) const override /// NOLINT - { - auto & col = static_cast(to); - auto & data_col = static_cast(col.getData()); - auto & offset_col = static_cast( - col.getOffsetsColumn() - ); - - data_col.reserve(data_col.size() + category_count); - - T sum_no = reinterpret_cast(place)[category_count * 2]; - T sum_yes = reinterpret_cast(place)[category_count * 2 + 1]; - - Float64 rev_no = 1. / sum_no; - Float64 rev_yes = 1. / sum_yes; - - for (size_t i : collections::range(0, category_count)) - { - T no = reinterpret_cast(place)[i * 2]; - T yes = reinterpret_cast(place)[i * 2 + 1]; - - data_col.insertValue((no * rev_no - yes * rev_yes) * (log(no * rev_no) - log(yes * rev_yes))); - } - - offset_col.insertValue(data_col.size()); - } -}; - -} diff --git a/src/AggregateFunctions/AggregateFunctionIf.cpp b/src/AggregateFunctions/AggregateFunctionIf.cpp index fa5e6b85a1e..0cf92585b77 100644 --- a/src/AggregateFunctions/AggregateFunctionIf.cpp +++ b/src/AggregateFunctions/AggregateFunctionIf.cpp @@ -278,6 +278,71 @@ public: } } + void addBatchSinglePlace( + size_t row_begin, size_t row_end, AggregateDataPtr __restrict place, const IColumn ** columns, Arena * arena, ssize_t) const final + { + std::unique_ptr final_null_flags = std::make_unique(row_end); + const size_t filter_column_num = number_of_arguments - 1; + + if (is_nullable[filter_column_num]) + { + const ColumnNullable * nullable_column = assert_cast(columns[filter_column_num]); + const IColumn & filter_column = nullable_column->getNestedColumn(); + const UInt8 * filter_null_map = nullable_column->getNullMapColumn().getData().data(); + const UInt8 * filter_values = assert_cast(filter_column).getData().data(); + + for (size_t i = row_begin; i < row_end; i++) + { + final_null_flags[i] = (null_is_skipped && filter_null_map[i]) || !filter_values[i]; + } + } + else + { + const IColumn * filter_column = columns[filter_column_num]; + const UInt8 * filter_values = assert_cast(filter_column)->getData().data(); + for (size_t i = row_begin; i < row_end; i++) + final_null_flags[i] = !filter_values[i]; + } + + const IColumn * nested_columns[number_of_arguments]; + for (size_t arg = 0; arg < number_of_arguments; arg++) + { + if (is_nullable[arg]) + { + const ColumnNullable & nullable_col = assert_cast(*columns[arg]); + if (null_is_skipped && (arg != filter_column_num)) + { + const ColumnUInt8 & nullmap_column = nullable_col.getNullMapColumn(); + const UInt8 * col_null_map = nullmap_column.getData().data(); + for (size_t r = row_begin; r < row_end; r++) + { + final_null_flags[r] |= col_null_map[r]; + } + } + nested_columns[arg] = &nullable_col.getNestedColumn(); + } + else + nested_columns[arg] = columns[arg]; + } + + bool at_least_one = false; + for (size_t i = row_begin; i < row_end; i++) + { + if (!final_null_flags[i]) + { + at_least_one = true; + break; + } + } + + if (at_least_one) + { + this->setFlag(place); + this->nested_function->addBatchSinglePlaceNotNull( + row_begin, row_end, this->nestedPlace(place), nested_columns, final_null_flags.get(), arena, -1); + } + } + #if USE_EMBEDDED_COMPILER void compileAdd(llvm::IRBuilderBase & builder, llvm::Value * aggregate_data_ptr, const DataTypes & arguments_types, const std::vector & argument_values) const override diff --git a/src/AggregateFunctions/AggregateFunctionMinMaxAny.h b/src/AggregateFunctions/AggregateFunctionMinMaxAny.h index d579a925f9d..ad633418ec3 100644 --- a/src/AggregateFunctions/AggregateFunctionMinMaxAny.h +++ b/src/AggregateFunctions/AggregateFunctionMinMaxAny.h @@ -492,7 +492,7 @@ public: void insertResultInto(IColumn & to) const { if (has()) - assert_cast(to).insertDataWithTerminatingZero(getData(), size); + assert_cast(to).insertData(getData(), size); else assert_cast(to).insertDefault(); } @@ -569,7 +569,7 @@ public: void change(const IColumn & column, size_t row_num, Arena * arena) { - changeImpl(assert_cast(column).getDataAtWithTerminatingZero(row_num), arena); + changeImpl(assert_cast(column).getDataAt(row_num), arena); } void change(const Self & to, Arena * arena) @@ -618,7 +618,7 @@ public: bool changeIfLess(const IColumn & column, size_t row_num, Arena * arena) { - if (!has() || assert_cast(column).getDataAtWithTerminatingZero(row_num) < getStringRef()) + if (!has() || assert_cast(column).getDataAt(row_num) < getStringRef()) { change(column, row_num, arena); return true; @@ -640,7 +640,7 @@ public: bool changeIfGreater(const IColumn & column, size_t row_num, Arena * arena) { - if (!has() || assert_cast(column).getDataAtWithTerminatingZero(row_num) > getStringRef()) + if (!has() || assert_cast(column).getDataAt(row_num) > getStringRef()) { change(column, row_num, arena); return true; @@ -667,7 +667,7 @@ public: bool isEqualTo(const IColumn & column, size_t row_num) const { - return has() && assert_cast(column).getDataAtWithTerminatingZero(row_num) == getStringRef(); + return has() && assert_cast(column).getDataAt(row_num) == getStringRef(); } static bool allocatesMemoryInArena() diff --git a/src/AggregateFunctions/AggregateFunctionNull.h b/src/AggregateFunctions/AggregateFunctionNull.h index ca284680800..1e2c9326142 100644 --- a/src/AggregateFunctions/AggregateFunctionNull.h +++ b/src/AggregateFunctions/AggregateFunctionNull.h @@ -414,6 +414,109 @@ public: this->nested_function->add(this->nestedPlace(place), nested_columns, row_num, arena); } + void addBatchSinglePlace( + size_t row_begin, + size_t row_end, + AggregateDataPtr __restrict place, + const IColumn ** columns, + Arena * arena, + ssize_t if_argument_pos) const final + { + /// We are going to merge all the flags into a single one to be able to call the nested batching functions + std::vector nullable_filters; + const IColumn * nested_columns[number_of_arguments]; + + std::unique_ptr final_flags = nullptr; + const UInt8 * final_flags_ptr = nullptr; + + if (if_argument_pos >= 0) + { + final_flags = std::make_unique(row_end); + final_flags_ptr = final_flags.get(); + + bool included_elements = 0; + const auto & flags = assert_cast(*columns[if_argument_pos]).getData(); + for (size_t i = row_begin; i < row_end; i++) + { + final_flags[i] = !flags.data()[i]; + included_elements += !!flags.data()[i]; + } + + if (included_elements == 0) + return; + if (included_elements != (row_end - row_begin)) + { + nullable_filters.push_back(final_flags_ptr); + } + } + + for (size_t i = 0; i < number_of_arguments; ++i) + { + if (is_nullable[i]) + { + const ColumnNullable & nullable_col = assert_cast(*columns[i]); + nested_columns[i] = &nullable_col.getNestedColumn(); + if constexpr (null_is_skipped) + { + const ColumnUInt8 & nullmap_column = nullable_col.getNullMapColumn(); + nullable_filters.push_back(nullmap_column.getData().data()); + } + } + else + { + nested_columns[i] = columns[i]; + } + } + + bool found_one = false; + + chassert(nullable_filters.size() > 0); /// We work under the assumption that we reach this because one argument was NULL + if (nullable_filters.size() == 1) + { + /// We can avoid making copies of the only filter but we still need to check that there is data to be added + final_flags_ptr = nullable_filters[0]; + for (size_t i = row_begin; i < row_end; i++) + { + if (!final_flags_ptr[i]) + { + found_one = true; + break; + } + } + } + else + { + if (!final_flags) + { + final_flags = std::make_unique(row_end); + final_flags_ptr = final_flags.get(); + } + + const size_t filter_start = nullable_filters[0] == final_flags_ptr ? 1 : 0; + for (size_t filter = filter_start; filter < nullable_filters.size(); filter++) + { + for (size_t i = row_begin; i < row_end; i++) + final_flags[i] |= nullable_filters[filter][i]; + } + + for (size_t i = row_begin; i < row_end; i++) + { + if (!final_flags_ptr[i]) + { + found_one = true; + break; + } + } + } + + if (!found_one) + return; // Nothing to do and nothing to mark + + this->setFlag(place); + this->nested_function->addBatchSinglePlaceNotNull( + row_begin, row_end, this->nestedPlace(place), nested_columns, final_flags_ptr, arena, -1); + } + #if USE_EMBEDDED_COMPILER diff --git a/src/AggregateFunctions/AggregateFunctionUniqUpTo.h b/src/AggregateFunctions/AggregateFunctionUniqUpTo.h index aecaecbe4bf..48b4c0f2c68 100644 --- a/src/AggregateFunctions/AggregateFunctionUniqUpTo.h +++ b/src/AggregateFunctions/AggregateFunctionUniqUpTo.h @@ -18,11 +18,6 @@ #include -#if !defined(__clang__) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Warray-bounds" -#endif - namespace DB { struct Settings; @@ -291,7 +286,3 @@ public: } - -#if !defined(__clang__) -#pragma GCC diagnostic pop -#endif diff --git a/src/AggregateFunctions/ReservoirSamplerDeterministic.h b/src/AggregateFunctions/ReservoirSamplerDeterministic.h index b92ba8cfd7b..a64c02e823b 100644 --- a/src/AggregateFunctions/ReservoirSamplerDeterministic.h +++ b/src/AggregateFunctions/ReservoirSamplerDeterministic.h @@ -165,11 +165,6 @@ public: sorted = false; } -#if !defined(__clang__) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wclass-memaccess" -#endif - void write(DB::WriteBuffer & buf) const { size_t size = samples.size(); @@ -193,10 +188,6 @@ public: } } -#if !defined(__clang__) -#pragma GCC diagnostic pop -#endif - private: /// We allocate some memory on the stack to avoid allocations when there are many objects with a small number of elements. using Element = std::pair; diff --git a/src/AggregateFunctions/UniquesHashSet.h b/src/AggregateFunctions/UniquesHashSet.h index 8648f6e2500..54503e356c2 100644 --- a/src/AggregateFunctions/UniquesHashSet.h +++ b/src/AggregateFunctions/UniquesHashSet.h @@ -424,14 +424,30 @@ public: alloc(new_size_degree); - for (size_t i = 0; i < m_size; ++i) + if (m_size <= 1) { - HashValue x = 0; - DB::readIntBinary(x, rb); - if (x == 0) - has_zero = true; - else - reinsertImpl(x); + for (size_t i = 0; i < m_size; ++i) + { + HashValue x = 0; + DB::readIntBinary(x, rb); + if (x == 0) + has_zero = true; + else + reinsertImpl(x); + } + } + else + { + auto hs = std::make_unique(m_size); + rb.readStrict(reinterpret_cast(hs.get()), m_size * sizeof(HashValue)); + + for (size_t i = 0; i < m_size; ++i) + { + if (hs[i] == 0) + has_zero = true; + else + reinsertImpl(hs[i]); + } } } @@ -458,11 +474,24 @@ public: resize(new_size_degree); } - for (size_t i = 0; i < rhs_size; ++i) + if (rhs_size <= 1) { - HashValue x = 0; - DB::readIntBinary(x, rb); - insertHash(x); + for (size_t i = 0; i < rhs_size; ++i) + { + HashValue x = 0; + DB::readIntBinary(x, rb); + insertHash(x); + } + } + else + { + auto hs = std::make_unique(rhs_size); + rb.readStrict(reinterpret_cast(hs.get()), rhs_size * sizeof(HashValue)); + + for (size_t i = 0; i < rhs_size; ++i) + { + insertHash(hs[i]); + } } } diff --git a/src/BridgeHelper/CatBoostLibraryBridgeHelper.cpp b/src/BridgeHelper/CatBoostLibraryBridgeHelper.cpp new file mode 100644 index 00000000000..b0ef9b91a28 --- /dev/null +++ b/src/BridgeHelper/CatBoostLibraryBridgeHelper.cpp @@ -0,0 +1,194 @@ +#include "CatBoostLibraryBridgeHelper.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + +CatBoostLibraryBridgeHelper::CatBoostLibraryBridgeHelper( + ContextPtr context_, + std::optional model_path_, + std::optional library_path_) + : LibraryBridgeHelper(context_->getGlobalContext()) + , model_path(model_path_) + , library_path(library_path_) +{ +} + +Poco::URI CatBoostLibraryBridgeHelper::getPingURI() const +{ + auto uri = createBaseURI(); + uri.setPath(PING_HANDLER); + return uri; +} + +Poco::URI CatBoostLibraryBridgeHelper::getMainURI() const +{ + auto uri = createBaseURI(); + uri.setPath(MAIN_HANDLER); + return uri; +} + + +Poco::URI CatBoostLibraryBridgeHelper::createRequestURI(const String & method) const +{ + auto uri = getMainURI(); + uri.addQueryParameter("version", std::to_string(LIBRARY_BRIDGE_PROTOCOL_VERSION)); + uri.addQueryParameter("method", method); + return uri; +} + +bool CatBoostLibraryBridgeHelper::bridgeHandShake() +{ + String result; + try + { + ReadWriteBufferFromHTTP buf(getPingURI(), Poco::Net::HTTPRequest::HTTP_GET, {}, http_timeouts, credentials); + readString(result, buf); + } + catch (...) + { + tryLogCurrentException(log); + return false; + } + + if (result != "1") + throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected message from library bridge: {}. Check that bridge and server have the same version.", result); + + return true; +} + +ExternalModelInfos CatBoostLibraryBridgeHelper::listModels() +{ + startBridgeSync(); + + ReadWriteBufferFromHTTP buf( + createRequestURI(CATBOOST_LIST_METHOD), + Poco::Net::HTTPRequest::HTTP_POST, + [](std::ostream &) {}, + http_timeouts, credentials); + + ExternalModelInfos result; + + UInt64 num_rows; + readIntBinary(num_rows, buf); + + for (UInt64 i = 0; i < num_rows; ++i) + { + ExternalModelInfo info; + + readStringBinary(info.model_path, buf); + readStringBinary(info.model_type, buf); + + UInt64 t; + readIntBinary(t, buf); + info.loading_start_time = std::chrono::system_clock::from_time_t(t); + + readIntBinary(t, buf); + info.loading_duration = std::chrono::milliseconds(t); + + result.push_back(info); + } + + return result; +} + +void CatBoostLibraryBridgeHelper::removeModel() +{ + startBridgeSync(); + + assert(model_path); + + ReadWriteBufferFromHTTP buf( + createRequestURI(CATBOOST_REMOVEMODEL_METHOD), + Poco::Net::HTTPRequest::HTTP_POST, + [this](std::ostream & os) + { + os << "model_path=" << escapeForFileName(*model_path); + }, + http_timeouts, credentials); + + String result; + readStringBinary(result, buf); + assert(result == "1"); +} + +void CatBoostLibraryBridgeHelper::removeAllModels() +{ + startBridgeSync(); + + ReadWriteBufferFromHTTP buf( + createRequestURI(CATBOOST_REMOVEALLMODELS_METHOD), + Poco::Net::HTTPRequest::HTTP_POST, + [](std::ostream &){}, + http_timeouts, credentials); + + String result; + readStringBinary(result, buf); + assert(result == "1"); +} + +size_t CatBoostLibraryBridgeHelper::getTreeCount() +{ + startBridgeSync(); + + assert(model_path && library_path); + + ReadWriteBufferFromHTTP buf( + createRequestURI(CATBOOST_GETTREECOUNT_METHOD), + Poco::Net::HTTPRequest::HTTP_POST, + [this](std::ostream & os) + { + os << "library_path=" << escapeForFileName(*library_path) << "&"; + os << "model_path=" << escapeForFileName(*model_path); + }, + http_timeouts, credentials); + + size_t result; + readIntBinary(result, buf); + return result; +} + +ColumnPtr CatBoostLibraryBridgeHelper::evaluate(const ColumnsWithTypeAndName & columns) +{ + startBridgeSync(); + + WriteBufferFromOwnString string_write_buf; + Block block(columns); + NativeWriter serializer(string_write_buf, /*client_revision*/ 0, block); + serializer.write(block); + + assert(model_path); + + ReadWriteBufferFromHTTP buf( + createRequestURI(CATBOOST_LIB_EVALUATE_METHOD), + Poco::Net::HTTPRequest::HTTP_POST, + [this, serialized = string_write_buf.str()](std::ostream & os) + { + os << "model_path=" << escapeForFileName(*model_path) << "&"; + os << "data=" << escapeForFileName(serialized); + }, + http_timeouts, credentials); + + NativeReader deserializer(buf, /*server_revision*/ 0); + Block block_read = deserializer.read(); + + return block_read.getColumns()[0]; +} + +} diff --git a/src/BridgeHelper/CatBoostLibraryBridgeHelper.h b/src/BridgeHelper/CatBoostLibraryBridgeHelper.h new file mode 100644 index 00000000000..91c94143147 --- /dev/null +++ b/src/BridgeHelper/CatBoostLibraryBridgeHelper.h @@ -0,0 +1,53 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ + +class CatBoostLibraryBridgeHelper : public LibraryBridgeHelper +{ +public: + static constexpr inline auto PING_HANDLER = "/catboost_ping"; + static constexpr inline auto MAIN_HANDLER = "/catboost_request"; + + explicit CatBoostLibraryBridgeHelper( + ContextPtr context_, + std::optional model_path_ = std::nullopt, + std::optional library_path_ = std::nullopt); + + ExternalModelInfos listModels(); + + void removeModel(); /// requires model_path + void removeAllModels(); + + size_t getTreeCount(); /// requires model_path and library_path + ColumnPtr evaluate(const ColumnsWithTypeAndName & columns); /// requires model_path + +protected: + Poco::URI getPingURI() const override; + + Poco::URI getMainURI() const override; + + bool bridgeHandShake() override; + +private: + static constexpr inline auto CATBOOST_LIST_METHOD = "catboost_list"; + static constexpr inline auto CATBOOST_REMOVEMODEL_METHOD = "catboost_removeModel"; + static constexpr inline auto CATBOOST_REMOVEALLMODELS_METHOD = "catboost_removeAllModels"; + static constexpr inline auto CATBOOST_GETTREECOUNT_METHOD = "catboost_GetTreeCount"; + static constexpr inline auto CATBOOST_LIB_EVALUATE_METHOD = "catboost_libEvaluate"; + + Poco::URI createRequestURI(const String & method) const; + + const std::optional model_path; + const std::optional library_path; +}; + +} diff --git a/src/BridgeHelper/IBridgeHelper.h b/src/BridgeHelper/IBridgeHelper.h index 5068e84f885..a3348c81b68 100644 --- a/src/BridgeHelper/IBridgeHelper.h +++ b/src/BridgeHelper/IBridgeHelper.h @@ -12,8 +12,8 @@ namespace DB { -/// Common base class for XDBC and Library bridge helpers. -/// Contains helper methods to check/start bridge sync. +/// Base class for server-side bridge helpers, e.g. xdbc-bridge and library-bridge. +/// Contains helper methods to check/start bridge sync class IBridgeHelper: protected WithContext { diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp index 465d4358e91..7d05cbb0681 100644 --- a/src/Client/ClientBase.cpp +++ b/src/Client/ClientBase.cpp @@ -1080,6 +1080,20 @@ bool ClientBase::receiveSampleBlock(Block & out, ColumnsDescription & columns_de } +void ClientBase::setInsertionTable(const ASTInsertQuery & insert_query) +{ + if (!global_context->hasInsertionTable() && insert_query.table) + { + String table = insert_query.table->as().shortName(); + if (!table.empty()) + { + String database = insert_query.database ? insert_query.database->as().shortName() : ""; + global_context->setInsertionTable(StorageID(database, table)); + } + } +} + + void ClientBase::processInsertQuery(const String & query_to_execute, ASTPtr parsed_query) { auto query = query_to_execute; @@ -1129,6 +1143,8 @@ void ClientBase::processInsertQuery(const String & query_to_execute, ASTPtr pars { /// If structure was received (thus, server has not thrown an exception), /// send our data with that structure. + setInsertionTable(parsed_insert_query); + sendData(sample, columns_description, parsed_query); receiveEndOfQuery(); } diff --git a/src/Client/ClientBase.h b/src/Client/ClientBase.h index 219d35d87cd..278056130fd 100644 --- a/src/Client/ClientBase.h +++ b/src/Client/ClientBase.h @@ -113,6 +113,8 @@ protected: std::vector & external_tables_arguments, std::vector & hosts_and_ports_arguments) = 0; + void setInsertionTable(const ASTInsertQuery & insert_query); + private: void receiveResult(ASTPtr parsed_query); diff --git a/src/Client/QueryFuzzer.cpp b/src/Client/QueryFuzzer.cpp index 9b404e7c5b7..5e231108bed 100644 --- a/src/Client/QueryFuzzer.cpp +++ b/src/Client/QueryFuzzer.cpp @@ -137,9 +137,41 @@ Field QueryFuzzer::fuzzField(Field field) break; } } - else if (type == Field::Types::Array || type == Field::Types::Tuple) + else if (type == Field::Types::Array) { - auto & arr = field.reinterpret(); + auto & arr = field.get(); + + if (fuzz_rand() % 5 == 0 && !arr.empty()) + { + size_t pos = fuzz_rand() % arr.size(); + arr.erase(arr.begin() + pos); + std::cerr << "erased\n"; + } + + if (fuzz_rand() % 5 == 0) + { + if (!arr.empty()) + { + size_t pos = fuzz_rand() % arr.size(); + arr.insert(arr.begin() + pos, fuzzField(arr[pos])); + std::cerr << fmt::format("inserted (pos {})\n", pos); + } + else + { + arr.insert(arr.begin(), getRandomField(0)); + std::cerr << "inserted (0)\n"; + } + + } + + for (auto & element : arr) + { + element = fuzzField(element); + } + } + else if (type == Field::Types::Tuple) + { + auto & arr = field.get(); if (fuzz_rand() % 5 == 0 && !arr.empty()) { diff --git a/src/Client/Suggest.cpp b/src/Client/Suggest.cpp index f8d41853566..552895e754d 100644 --- a/src/Client/Suggest.cpp +++ b/src/Client/Suggest.cpp @@ -187,9 +187,8 @@ void Suggest::fillWordsFromBlock(const Block & block) Words new_words; new_words.reserve(rows); for (size_t i = 0; i < rows; ++i) - { - new_words.emplace_back(column.getDataAt(i).toString()); - } + new_words.emplace_back(column[i].get()); + addWords(std::move(new_words)); } diff --git a/src/Columns/ColumnArray.cpp b/src/Columns/ColumnArray.cpp index 7bddfc14707..bb56baf9216 100644 --- a/src/Columns/ColumnArray.cpp +++ b/src/Columns/ColumnArray.cpp @@ -151,23 +151,24 @@ void ColumnArray::get(size_t n, Field & res) const StringRef ColumnArray::getDataAt(size_t n) const { + assert(n < size()); + /** Returns the range of memory that covers all elements of the array. * Works for arrays of fixed length values. - * For arrays of strings and arrays of arrays, the resulting chunk of memory may not be one-to-one correspondence with the elements, - * since it contains only the data laid in succession, but not the offsets. */ - size_t offset_of_first_elem = offsetAt(n); - StringRef first = getData().getDataAtWithTerminatingZero(offset_of_first_elem); + /// We are using pointer arithmetic on the addresses of the array elements. + if (!data->isFixedAndContiguous()) + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method getDataAt is not supported for {}", getName()); size_t array_size = sizeAt(n); if (array_size == 0) - return StringRef(first.data, 0); + return StringRef(nullptr, 0); - size_t offset_of_last_elem = getOffsets()[n] - 1; - StringRef last = getData().getDataAtWithTerminatingZero(offset_of_last_elem); + size_t offset_of_first_elem = offsetAt(n); + StringRef first = getData().getDataAt(offset_of_first_elem); - return StringRef(first.data, last.data + last.size - first.data); + return StringRef(first.data, first.size * array_size); } @@ -183,7 +184,7 @@ void ColumnArray::insertData(const char * pos, size_t length) /** Similarly - only for arrays of fixed length values. */ if (!data->isFixedAndContiguous()) - throw Exception("Method insertData is not supported for " + getName(), ErrorCodes::NOT_IMPLEMENTED); + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method insertData is not supported for {}", getName()); size_t field_size = data->sizeOfValueIfFixed(); diff --git a/src/Columns/ColumnConst.h b/src/Columns/ColumnConst.h index 99a230720a4..3b15b7239b9 100644 --- a/src/Columns/ColumnConst.h +++ b/src/Columns/ColumnConst.h @@ -81,11 +81,6 @@ public: return data->getDataAt(0); } - StringRef getDataAtWithTerminatingZero(size_t) const override - { - return data->getDataAtWithTerminatingZero(0); - } - UInt64 get64(size_t) const override { return data->get64(0); diff --git a/src/Columns/ColumnLowCardinality.h b/src/Columns/ColumnLowCardinality.h index 7cd226c4c11..4786c57f8a5 100644 --- a/src/Columns/ColumnLowCardinality.h +++ b/src/Columns/ColumnLowCardinality.h @@ -59,10 +59,6 @@ public: void get(size_t n, Field & res) const override { getDictionary().get(getIndexes().getUInt(n), res); } StringRef getDataAt(size_t n) const override { return getDictionary().getDataAt(getIndexes().getUInt(n)); } - StringRef getDataAtWithTerminatingZero(size_t n) const override - { - return getDictionary().getDataAtWithTerminatingZero(getIndexes().getUInt(n)); - } bool isDefaultAt(size_t n) const override { return getDictionary().isDefaultAt(getIndexes().getUInt(n)); } UInt64 get64(size_t n) const override { return getDictionary().get64(getIndexes().getUInt(n)); } diff --git a/src/Columns/ColumnString.h b/src/Columns/ColumnString.h index 361b792df55..863b080c588 100644 --- a/src/Columns/ColumnString.h +++ b/src/Columns/ColumnString.h @@ -108,24 +108,12 @@ public: return StringRef(&chars[offsetAt(n)], sizeAt(n) - 1); } - StringRef getDataAtWithTerminatingZero(size_t n) const override - { - assert(n < size()); - return StringRef(&chars[offsetAt(n)], sizeAt(n)); - } - bool isDefaultAt(size_t n) const override { assert(n < size()); return sizeAt(n) == 1; } -/// Suppress gcc 7.3.1 warning: '*((void*)& +8)' may be used uninitialized in this function -#if !defined(__clang__) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" -#endif - void insert(const Field & x) override { const String & s = x.get(); @@ -138,10 +126,6 @@ public: offsets.push_back(new_size); } -#if !defined(__clang__) -#pragma GCC diagnostic pop -#endif - void insertFrom(const IColumn & src_, size_t n) override { const ColumnString & src = assert_cast(src_); @@ -177,17 +161,6 @@ public: offsets.push_back(new_size); } - /// Like getData, but inserting data should be zero-ending (i.e. length is 1 byte greater than real string size). - void insertDataWithTerminatingZero(const char * pos, size_t length) - { - const size_t old_size = chars.size(); - const size_t new_size = old_size + length; - - chars.resize(new_size); - memcpy(chars.data() + old_size, pos, length); - offsets.push_back(new_size); - } - void popBack(size_t n) override { size_t nested_n = offsets.back() - offsetAt(offsets.size() - n); diff --git a/src/Columns/ColumnUnique.h b/src/Columns/ColumnUnique.h index d3ab87410f3..53763e94b0d 100644 --- a/src/Columns/ColumnUnique.h +++ b/src/Columns/ColumnUnique.h @@ -1,4 +1,5 @@ #pragma once + #include #include #include @@ -7,16 +8,17 @@ #include #include #include +#include #include #include #include #include -#include +#include +#include #include -#include "Columns/ColumnConst.h" namespace DB @@ -70,10 +72,6 @@ public: void get(size_t n, Field & res) const override { getNestedColumn()->get(n, res); } bool isDefaultAt(size_t n) const override { return n == 0; } StringRef getDataAt(size_t n) const override { return getNestedColumn()->getDataAt(n); } - StringRef getDataAtWithTerminatingZero(size_t n) const override - { - return getNestedColumn()->getDataAtWithTerminatingZero(n); - } UInt64 get64(size_t n) const override { return getNestedColumn()->get64(n); } UInt64 getUInt(size_t n) const override { return getNestedColumn()->getUInt(n); } Int64 getInt(size_t n) const override { return getNestedColumn()->getInt(n); } @@ -309,17 +307,52 @@ size_t ColumnUnique::getNullValueIndex() const return 0; } + +namespace +{ + class FieldVisitorGetData : public StaticVisitor<> + { + public: + StringRef res; + + [[noreturn]] static void throwUnsupported() + { + throw Exception("Unsupported field type", ErrorCodes::LOGICAL_ERROR); + } + + [[noreturn]] void operator() (const Null &) { throwUnsupported(); } + [[noreturn]] void operator() (const Array &) { throwUnsupported(); } + [[noreturn]] void operator() (const Tuple &) { throwUnsupported(); } + [[noreturn]] void operator() (const Map &) { throwUnsupported(); } + [[noreturn]] void operator() (const Object &) { throwUnsupported(); } + [[noreturn]] void operator() (const AggregateFunctionStateData &) { throwUnsupported(); } + void operator() (const String & x) { res = {x.data(), x.size()}; } + void operator() (const UInt64 & x) { res = {reinterpret_cast(&x), sizeof(x)}; } + void operator() (const UInt128 & x) { res = {reinterpret_cast(&x), sizeof(x)}; } + void operator() (const UInt256 & x) { res = {reinterpret_cast(&x), sizeof(x)}; } + void operator() (const Int64 & x) { res = {reinterpret_cast(&x), sizeof(x)}; } + void operator() (const Int128 & x) { res = {reinterpret_cast(&x), sizeof(x)}; } + void operator() (const Int256 & x) { res = {reinterpret_cast(&x), sizeof(x)}; } + void operator() (const UUID & x) { res = {reinterpret_cast(&x), sizeof(x)}; } + void operator() (const Float64 & x) { res = {reinterpret_cast(&x), sizeof(x)}; } + void operator() (const DecimalField & x) { res = {reinterpret_cast(&x), sizeof(x)}; } + void operator() (const DecimalField & x) { res = {reinterpret_cast(&x), sizeof(x)}; } + void operator() (const DecimalField & x) { res = {reinterpret_cast(&x), sizeof(x)}; } + void operator() (const DecimalField & x) { res = {reinterpret_cast(&x), sizeof(x)}; } + void operator() (const bool & x) { res = {reinterpret_cast(&x), sizeof(x)}; } + }; +} + + template size_t ColumnUnique::uniqueInsert(const Field & x) { if (x.isNull()) return getNullValueIndex(); - if (valuesHaveFixedSize()) - return uniqueInsertData(&x.reinterpret(), size_of_value_if_fixed); - - const auto & val = x.get(); - return uniqueInsertData(val.data(), val.size()); + FieldVisitorGetData visitor; + applyVisitor(visitor, x); + return uniqueInsertData(visitor.res.data, visitor.res.size); } template diff --git a/src/Columns/IColumn.h b/src/Columns/IColumn.h index 974925d247e..d11bc9d435d 100644 --- a/src/Columns/IColumn.h +++ b/src/Columns/IColumn.h @@ -106,13 +106,6 @@ public: /// Is used to optimize some computations (in aggregation, for example). [[nodiscard]] virtual StringRef getDataAt(size_t n) const = 0; - /// Like getData, but has special behavior for columns that contain variable-length strings. - /// Returns zero-ending memory chunk (i.e. its size is 1 byte longer). - [[nodiscard]] virtual StringRef getDataAtWithTerminatingZero(size_t n) const - { - return getDataAt(n); - } - /// If column stores integers, it returns n-th element transformed to UInt64 using static_cast. /// If column stores floating point numbers, bits of n-th elements are copied to lower bits of UInt64, the remaining bits are zeros. /// Is used to optimize some computations (in aggregation, for example). diff --git a/src/Columns/tests/gtest_column_object.cpp b/src/Columns/tests/gtest_column_object.cpp index e1ad949f6a8..f9b6ff16b71 100644 --- a/src/Columns/tests/gtest_column_object.cpp +++ b/src/Columns/tests/gtest_column_object.cpp @@ -1,8 +1,11 @@ #include #include +#include +#include #include #include #include +#include #include #include @@ -118,3 +121,36 @@ TEST(ColumnObject, InsertRangeFrom) checkFieldsAreEqual(subcolumn_dst, fields_dst); } } + +TEST(ColumnObject, Unflatten) +{ + auto check_empty_tuple = [](const auto & type, const auto & column) + { + const auto & type_tuple = assert_cast(*type); + const auto & column_tuple = assert_cast(*column); + + ASSERT_EQ(type_tuple.getElements().size(), 1); + ASSERT_EQ(type_tuple.getElements()[0]->getName(), "UInt8"); + ASSERT_EQ(type_tuple.getElementNames()[0], ColumnObject::COLUMN_NAME_DUMMY); + + ASSERT_EQ(column_tuple.getColumns().size(), 1); + ASSERT_EQ(column_tuple.getColumns()[0]->getName(), "UInt8"); + }; + + { + auto column_object = ColumnObject::create(false); + auto [column, type] = unflattenObjectToTuple(*column_object); + + check_empty_tuple(type, column); + ASSERT_EQ(column->size(), 0); + } + + { + auto column_object = ColumnObject::create(false); + column_object->insertManyDefaults(5); + auto [column, type] = unflattenObjectToTuple(*column_object); + + check_empty_tuple(type, column); + ASSERT_EQ(column->size(), 5); + } +} diff --git a/src/Common/Allocator.h b/src/Common/Allocator.h index 06ccbed4064..c348eaea006 100644 --- a/src/Common/Allocator.h +++ b/src/Common/Allocator.h @@ -281,14 +281,6 @@ private: #endif }; -/** When using AllocatorWithStackMemory, located on the stack, - * GCC 4.9 mistakenly assumes that we can call `free` from a pointer to the stack. - * In fact, the combination of conditions inside AllocatorWithStackMemory does not allow this. - */ -#if !defined(__clang__) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wfree-nonheap-object" -#endif /** Allocator with optimization to place small memory ranges in automatic memory. */ @@ -366,7 +358,3 @@ extern template class Allocator; extern template class Allocator; extern template class Allocator; extern template class Allocator; - -#if !defined(__clang__) -#pragma GCC diagnostic pop -#endif diff --git a/src/Common/ArrayCache.h b/src/Common/ArrayCache.h index f01ff94e38b..79aeddb09df 100644 --- a/src/Common/ArrayCache.h +++ b/src/Common/ArrayCache.h @@ -722,5 +722,3 @@ public: return res; } }; - -template constexpr size_t ArrayCache::min_chunk_size; diff --git a/src/Common/Base64.cpp b/src/Common/Base64.cpp new file mode 100644 index 00000000000..74ce979b5b1 --- /dev/null +++ b/src/Common/Base64.cpp @@ -0,0 +1,33 @@ +#include + +#include +#include +#include +#include + +#include + +namespace DB +{ + +std::string base64Encode(const std::string & decoded, bool url_encoding) +{ + std::ostringstream ostr; // STYLE_CHECK_ALLOW_STD_STRING_STREAM + ostr.exceptions(std::ios::failbit); + Poco::Base64Encoder encoder(ostr, url_encoding ? Poco::BASE64_URL_ENCODING : 0); + encoder.rdbuf()->setLineLength(0); + encoder << decoded; + encoder.close(); + return ostr.str(); +} + +std::string base64Decode(const std::string & encoded, bool url_encoding) +{ + std::string decoded; + Poco::MemoryInputStream istr(encoded.data(), encoded.size()); + Poco::Base64Decoder decoder(istr, url_encoding ? Poco::BASE64_URL_ENCODING : 0); + Poco::StreamCopier::copyToString(decoder, decoded); + return decoded; +} + +} diff --git a/src/Common/Base64.h b/src/Common/Base64.h new file mode 100644 index 00000000000..963d3acb48f --- /dev/null +++ b/src/Common/Base64.h @@ -0,0 +1,12 @@ +#pragma once + +#include + +namespace DB +{ + +std::string base64Encode(const std::string & decoded, bool url_encoding = false); + +std::string base64Decode(const std::string & encoded, bool url_encoding = false); + +} diff --git a/src/Common/DateLUTImpl.h b/src/Common/DateLUTImpl.h index f5504749684..564d09aff6e 100644 --- a/src/Common/DateLUTImpl.h +++ b/src/Common/DateLUTImpl.h @@ -29,13 +29,6 @@ #define DATE_LUT_ADD ((1970 - DATE_LUT_MIN_YEAR) * 366L * 86400) -#if defined(__PPC__) -#if !defined(__clang__) -#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" -#endif -#endif - - /// Flags for toYearWeek() function. enum class WeekModeFlag : UInt8 { @@ -1445,9 +1438,3 @@ public: return s; } }; - -#if defined(__PPC__) -#if !defined(__clang__) -#pragma GCC diagnostic pop -#endif -#endif diff --git a/src/Common/Dwarf.cpp b/src/Common/Dwarf.cpp index c8ffe7d46a8..a912c49d2ae 100644 --- a/src/Common/Dwarf.cpp +++ b/src/Common/Dwarf.cpp @@ -26,6 +26,7 @@ #include #define DW_CHILDREN_no 0 + #define DW_FORM_addr 1 #define DW_FORM_block1 0x0a #define DW_FORM_block2 3 @@ -51,6 +52,25 @@ #define DW_FORM_string 0x08 #define DW_FORM_strp 0x0e #define DW_FORM_indirect 0x16 +#define DW_FORM_strx 0x1a +#define DW_FORM_addrx 0x1b +#define DW_FORM_ref_sup4 0x1c +#define DW_FORM_strp_sup 0x1d +#define DW_FORM_data16 0x1e +#define DW_FORM_line_strp 0x1f +#define DW_FORM_implicit_const 0x21 +#define DW_FORM_rnglistx 0x23 +#define DW_FORM_loclistx 0x22 +#define DW_FORM_ref_sup8 0x24 +#define DW_FORM_strx1 0x25 +#define DW_FORM_strx2 0x26 +#define DW_FORM_strx3 0x27 +#define DW_FORM_strx4 0x28 +#define DW_FORM_addrx1 0x29 +#define DW_FORM_addrx2 0x2a +#define DW_FORM_addrx3 0x2b +#define DW_FORM_addrx4 0x2c + #define DW_TAG_compile_unit 0x11 #define DW_TAG_subprogram 0x2e #define DW_TAG_try_block 0x32 @@ -58,6 +78,7 @@ #define DW_TAG_entry_point 0x03 #define DW_TAG_common_block 0x1a #define DW_TAG_lexical_block 0x0b + #define DW_AT_stmt_list 0x10 #define DW_AT_comp_dir 0x1b #define DW_AT_name 0x03 @@ -70,6 +91,13 @@ #define DW_AT_call_file 0x58 #define DW_AT_linkage_name 0x6e #define DW_AT_specification 0x47 +#define DW_AT_str_offsets_base 0x72 +#define DW_AT_addr_base 0x73 +#define DW_AT_rnglists_base 0x74 +#define DW_AT_loclists_base 0x8c +#define DW_AT_GNU_ranges_base 0x2132 +#define DW_AT_GNU_addr_base 0x2133 + #define DW_LNE_define_file 0x03 #define DW_LNS_copy 0x01 #define DW_LNS_advance_pc 0x02 @@ -87,6 +115,21 @@ #define DW_LNE_set_address 0x02 #define DW_LNE_set_discriminator 0x04 +#define DW_LNCT_path 0x1 +#define DW_LNCT_directory_index 0x2 +#define DW_LNCT_timestamp 0x3 +#define DW_LNCT_size 0x4 +#define DW_LNCT_MD5 0x5 + +#define DW_RLE_end_of_list 0x0 +#define DW_RLE_base_addressx 0x1 +#define DW_RLE_startx_endx 0x2 +#define DW_RLE_startx_length 0x3 +#define DW_RLE_offset_pair 0x4 +#define DW_RLE_base_address 0x5 +#define DW_RLE_start_end 0x6 +#define DW_RLE_start_length 0x7 + namespace DB { @@ -97,9 +140,31 @@ namespace ErrorCodes } -Dwarf::Dwarf(const std::shared_ptr & elf) : elf_(elf) +Dwarf::Dwarf(const std::shared_ptr & elf) + : elf_(elf) + , abbrev_(getSection(".debug_abbrev")) + , addr_(getSection(".debug_addr")) + , aranges_(getSection(".debug_aranges")) + , info_(getSection(".debug_info")) + , line_(getSection(".debug_line")) + , line_str_(getSection(".debug_line_str")) + , loclists_(getSection(".debug_loclists")) + , ranges_(getSection(".debug_ranges")) + , rnglists_(getSection(".debug_rnglists")) + , str_(getSection(".debug_str")) + , str_offsets_(getSection(".debug_str_offsets")) { - init(); + // Optional sections: + // - debugAranges_: for fast address range lookup. + // If missing .debug_info can be used - but it's much slower (linear + // scan). + // - debugRanges_ (DWARF 4) / debugRnglists_ (DWARF 5): non-contiguous + // address ranges of debugging information entries. + // Used for inline function address lookup. + if (info_.empty() || abbrev_.empty() || line_.empty() || str_.empty()) + { + elf_ = nullptr; + } } Dwarf::Section::Section(std::string_view d) : is64_bit(false), data(d) @@ -107,7 +172,7 @@ Dwarf::Section::Section(std::string_view d) : is64_bit(false), data(d) } -#define SAFE_CHECK(cond, message) do { if (!(cond)) throw Exception(message, ErrorCodes::CANNOT_PARSE_DWARF); } while (false) +#define SAFE_CHECK(cond, ...) do { if (!(cond)) throw Exception(ErrorCodes::CANNOT_PARSE_DWARF, __VA_ARGS__); } while (false) namespace @@ -124,13 +189,24 @@ template requires std::is_trivial_v && std::is_standard_layout_v T read(std::string_view & sp) { - SAFE_CHECK(sp.size() >= sizeof(T), fmt::format("underflow: expected bytes {}, got bytes {}", sizeof(T), sp.size())); + SAFE_CHECK(sp.size() >= sizeof(T), "underflow: expected bytes {}, got bytes {}", sizeof(T), sp.size()); T x; memcpy(&x, sp.data(), sizeof(T)); sp.remove_prefix(sizeof(T)); return x; } +// Read (bitwise) an unsigned number of N bytes (N in 1, 2, 3, 4). +template +uint64_t readU64(std::string_view & sp) +{ + SAFE_CHECK(sp.size() >= N, "underflow"); + uint64_t x = 0; + memcpy(&x, sp.data(), N); + sp.remove_prefix(N); + return x; +} + // Read ULEB (unsigned) varint value; algorithm from the DWARF spec uint64_t readULEB(std::string_view & sp, uint8_t & shift, uint8_t & val) { @@ -168,9 +244,9 @@ int64_t readSLEB(std::string_view & sp) } // Read a value of "section offset" type, which may be 4 or 8 bytes -uint64_t readOffset(std::string_view & sp, bool is64Bit) +uint64_t readOffset(std::string_view & sp, bool is64_bit) { - return is64Bit ? read(sp) : read(sp); + return is64_bit ? read(sp) : read(sp); } // Read "len" bytes @@ -192,6 +268,15 @@ std::string_view readNullTerminated(std::string_view & sp) return ret; } +// Get a string from the section +std::string_view getStringFromStringSection(std::string_view section, uint64_t offset) +{ + SAFE_CHECK(offset < section.size(), "invalid section offset"); + std::string_view sp(section); + sp.remove_prefix(offset); + return readNullTerminated(sp); +} + // Skip over padding until sp.data() - start is a multiple of alignment void skipPadding(std::string_view & sp, const char * start, size_t alignment) { @@ -359,38 +444,18 @@ bool Dwarf::Section::next(std::string_view & chunk) return true; } -bool Dwarf::getSection(const char * name, std::string_view * section) const +std::string_view Dwarf::getSection(const char * name) const { std::optional elf_section = elf_->findSectionByName(name); if (!elf_section) - return false; + return {}; #ifdef SHF_COMPRESSED if (elf_section->header.sh_flags & SHF_COMPRESSED) - return false; + return {}; #endif - *section = { elf_section->begin(), elf_section->size()}; - return true; -} - -void Dwarf::init() -{ - // Make sure that all .debug_* sections exist - if (!getSection(".debug_info", &info_) - || !getSection(".debug_abbrev", &abbrev_) - || !getSection(".debug_line", &line_) - || !getSection(".debug_str", &strings_)) - { - elf_.reset(); - return; - } - - // Optional: fast address range lookup. If missing .debug_info can - // be used - but it's much slower (linear scan). - getSection(".debug_aranges", &aranges_); - - getSection(".debug_ranges", &ranges_); + return { elf_section->begin(), elf_section->size()}; } // static @@ -473,7 +538,7 @@ size_t Dwarf::forEachAttribute(const CompilationUnit & cu, const Die & die, std: auto values = std::string_view{info_.data() + die.offset + die.attr_offset, cu.offset + cu.size - die.offset - die.attr_offset}; while (auto spec = readAttributeSpec(attrs)) { - auto attr = readAttribute(die, spec, values); + auto attr = readAttribute(cu, die, spec, values); if (!f(attr)) { return static_cast(-1); @@ -482,8 +547,49 @@ size_t Dwarf::forEachAttribute(const CompilationUnit & cu, const Die & die, std: return values.data() - info_.data(); } -Dwarf::Attribute Dwarf::readAttribute(const Die & die, AttributeSpec spec, std::string_view & info) const +Dwarf::Attribute Dwarf::readAttribute(const CompilationUnit & cu, + const Die & die, + AttributeSpec spec, + std::string_view & info) const { + // DWARF 5 introduces new FORMs whose values are relative to some base attrs: + // DW_AT_str_offsets_base, DW_AT_rnglists_base, DW_AT_addr_base. + // Debug Fission DWARF 4 uses GNU DW_AT_GNU_ranges_base & DW_AT_GNU_addr_base. + // + // The order in which attributes appear in a CU is not defined. + // The DW_AT_*_base attrs may appear after attributes that need them. + // The DW_AT_*_base attrs are CU specific; so we read them just after + // reading the CU header. During this first pass return empty values + // when encountering a FORM that depends on DW_AT_*_base. + auto get_string_using_offset_table = [&](uint64_t index) + { + if (!cu.str_offsets_base.has_value()) + { + return std::string_view(); + } + // DWARF 5: 7.26 String Offsets Table + // The DW_AT_str_offsets_base attribute points to the first entry following + // the header. The entries are indexed sequentially from this base entry, + // starting from 0. + auto sp = str_offsets_.substr(*cu.str_offsets_base + index * (cu.is64Bit ? sizeof(uint64_t) : sizeof(uint32_t))); + uint64_t str_offset = readOffset(sp, cu.is64Bit); + return getStringFromStringSection(str_, str_offset); + }; + + auto read_debug_addr = [&](uint64_t index) + { + if (!cu.addr_base.has_value()) + { + return uint64_t(0); + } + // DWARF 5: 7.27 Address Table + // The DW_AT_addr_base attribute points to the first entry following the + // header. The entries are indexed sequentially from this base entry, + // starting from 0. + auto sp = addr_.substr(*cu.addr_base + index * sizeof(uint64_t)); + return read(sp); + }; + switch (spec.form) { case DW_FORM_addr: @@ -517,7 +623,7 @@ Dwarf::Attribute Dwarf::readAttribute(const Die & die, AttributeSpec spec, std:: case DW_FORM_ref_sig8: return {spec, die, read(info)}; case DW_FORM_sdata: - return {spec, die, uint64_t(readSLEB(info))}; + return {spec, die, static_cast(readSLEB(info))}; case DW_FORM_udata: [[fallthrough]]; case DW_FORM_ref_udata: @@ -525,7 +631,7 @@ Dwarf::Attribute Dwarf::readAttribute(const Die & die, AttributeSpec spec, std:: case DW_FORM_flag: return {spec, die, read(info)}; case DW_FORM_flag_present: - return {spec, die, 1u}; + return {spec, die, 1ULL}; case DW_FORM_sec_offset: [[fallthrough]]; case DW_FORM_ref_addr: @@ -533,49 +639,215 @@ Dwarf::Attribute Dwarf::readAttribute(const Die & die, AttributeSpec spec, std:: case DW_FORM_string: return {spec, die, readNullTerminated(info)}; case DW_FORM_strp: - return {spec, die, getStringFromStringSection(readOffset(info, die.is64Bit))}; + return {spec, die, getStringFromStringSection(str_, readOffset(info, die.is64Bit))}; case DW_FORM_indirect: // form is explicitly specified // Update spec with the actual FORM. spec.form = readULEB(info); - return readAttribute(die, spec, info); + return readAttribute(cu, die, spec, info); + + // DWARF 5: + case DW_FORM_implicit_const: // form is explicitly specified + // For attributes with this form, the attribute specification contains a + // third part, which is a signed LEB128 number. The value of this number + // is used as the value of the attribute, and no value is stored in the + // .debug_info section. + return {spec, die, static_cast(spec.implicitConst)}; + + case DW_FORM_addrx: + return {spec, die, read_debug_addr(readULEB(info))}; + case DW_FORM_addrx1: + return {spec, die, read_debug_addr(readU64<1>(info))}; + case DW_FORM_addrx2: + return {spec, die, read_debug_addr(readU64<2>(info))}; + case DW_FORM_addrx3: + return {spec, die, read_debug_addr(readU64<3>(info))}; + case DW_FORM_addrx4: + return {spec, die, read_debug_addr(readU64<4>(info))}; + + case DW_FORM_line_strp: + return {spec, die, getStringFromStringSection(line_str_, readOffset(info, die.is64Bit))}; + + case DW_FORM_strx: + return {spec, die, get_string_using_offset_table(readULEB(info))}; + case DW_FORM_strx1: + return {spec, die, get_string_using_offset_table(readU64<1>(info))}; + case DW_FORM_strx2: + return {spec, die, get_string_using_offset_table(readU64<2>(info))}; + case DW_FORM_strx3: + return {spec, die, get_string_using_offset_table(readU64<3>(info))}; + case DW_FORM_strx4: + return {spec, die, get_string_using_offset_table(readU64<4>(info))}; + + case DW_FORM_rnglistx: { + auto index = readULEB(info); + if (!cu.rnglists_base.has_value()) + { + return {spec, die, 0ULL}; + } + const uint64_t offset_size = cu.is64Bit ? sizeof(uint64_t) : sizeof(uint32_t); + auto sp = rnglists_.substr(*cu.rnglists_base + index * offset_size); + auto offset = readOffset(sp, cu.is64Bit); + return {spec, die, *cu.rnglists_base + offset}; + } + + case DW_FORM_loclistx: { + auto index = readULEB(info); + if (!cu.loclists_base.has_value()) + { + return {spec, die, 0ULL}; + } + const uint64_t offset_size = cu.is64Bit ? sizeof(uint64_t) : sizeof(uint32_t); + auto sp = loclists_.substr(*cu.loclists_base + index * offset_size); + auto offset = readOffset(sp, cu.is64Bit); + return {spec, die, *cu.loclists_base + offset}; + } + + case DW_FORM_data16: + return {spec, die, readBytes(info, 16)}; + + case DW_FORM_ref_sup4: + case DW_FORM_ref_sup8: + case DW_FORM_strp_sup: + SAFE_CHECK(false, "Unexpected DWARF5 supplimentary object files"); + default: SAFE_CHECK(false, "invalid attribute form"); } - - return {spec, die, 0u}; + return {spec, die, 0ULL}; } // static Dwarf::AttributeSpec Dwarf::readAttributeSpec(std::string_view & sp) { - return {readULEB(sp), readULEB(sp)}; + Dwarf::AttributeSpec spec; + spec.name = readULEB(sp); + spec.form = readULEB(sp); + if (spec.form == DW_FORM_implicit_const) + { + spec.implicitConst = readSLEB(sp); + } + return spec; } -// static -Dwarf::CompilationUnit Dwarf::getCompilationUnit(std::string_view info, uint64_t offset) +Dwarf::CompilationUnit Dwarf::getCompilationUnit(uint64_t offset) const { - SAFE_CHECK(offset < info.size(), "unexpected offset"); + // SAFE_CHECK(offset < info_.size(), "unexpected offset"); CompilationUnit cu; - std::string_view chunk(info); + std::string_view chunk(info_); cu.offset = offset; chunk.remove_prefix(offset); + // 1) unit_length auto initial_length = read(chunk); cu.is64Bit = (initial_length == uint32_t(-1)); cu.size = cu.is64Bit ? read(chunk) : initial_length; SAFE_CHECK(cu.size <= chunk.size(), "invalid chunk size"); cu.size += cu.is64Bit ? 12 : 4; + // 2) version cu.version = read(chunk); - SAFE_CHECK(cu.version >= 2 && cu.version <= 4, "invalid info version"); - cu.abbrev_offset = readOffset(chunk, cu.is64Bit); - cu.addr_size = read(chunk); - SAFE_CHECK(cu.addr_size == sizeof(uintptr_t), "invalid address size"); + SAFE_CHECK(cu.version >= 2 && cu.version <= 5, "invalid info version"); - cu.first_die = chunk.data() - info.data(); + if (cu.version == 5) + { + // DWARF5: 7.5.1.1 Full and Partial Compilation Unit Headers + // 3) unit_type (new DWARF 5) + cu.unit_type = read(chunk); + if (cu.unit_type != DW_UT_compile && cu.unit_type != DW_UT_skeleton) + { + return cu; + } + // 4) address_size + cu.addr_size = read(chunk); + SAFE_CHECK(cu.addr_size == sizeof(uintptr_t), "invalid address size"); + + // 5) debug_abbrev_offset + cu.abbrev_offset = readOffset(chunk, cu.is64Bit); + + if (cu.unit_type == DW_UT_skeleton) + { + // 6) dwo_id + read(chunk); + } + } + else + { + // DWARF4 has a single type of unit in .debug_info + cu.unit_type = DW_UT_compile; + // 3) debug_abbrev_offset + cu.abbrev_offset = readOffset(chunk, cu.is64Bit); + // 4) address_size + cu.addr_size = read(chunk); + SAFE_CHECK(cu.addr_size == sizeof(uintptr_t), "invalid address size"); + } + cu.first_die = chunk.data() - info_.data(); + if (cu.version < 5) + { + return cu; + } + + Die die = getDieAtOffset(cu, cu.first_die); + if (die.abbr.tag != DW_TAG_compile_unit) + { + return cu; + } + + // Read the DW_AT_*_base attributes. + // Attributes which use FORMs relative to these base attrs + // will not have valid values during this first pass! + forEachAttribute( + cu, + die, + [&](const Attribute & attr) + { + switch (attr.spec.name) + { + case DW_AT_addr_base: + case DW_AT_GNU_addr_base: + cu.addr_base = std::get(attr.attr_value); + break; + case DW_AT_loclists_base: + cu.loclists_base = std::get(attr.attr_value); + break; + case DW_AT_rnglists_base: + case DW_AT_GNU_ranges_base: + cu.rnglists_base = std::get(attr.attr_value); + break; + case DW_AT_str_offsets_base: + cu.str_offsets_base = std::get(attr.attr_value); + break; + } + return true; // continue forEachAttribute + }); return cu; } +// Finds the Compilation Unit starting at offset. +Dwarf::CompilationUnit Dwarf::findCompilationUnit(uint64_t targetOffset) const +{ + // SAFE_CHECK(targetOffset < info_.size(), "unexpected target address"); + uint64_t offset = 0; + while (offset < info_.size()) + { + std::string_view chunk(info_); + chunk.remove_prefix(offset); + + auto initial_length = read(chunk); + auto is64_bit = (initial_length == static_cast(-1)); + auto size = is64_bit ? read(chunk) : initial_length; + SAFE_CHECK(size <= chunk.size(), "invalid chunk size"); + size += is64_bit ? 12 : 4; + + if (offset + size > targetOffset) + { + break; + } + offset += size; + } + return getCompilationUnit(offset); +} + + Dwarf::DIEAbbreviation Dwarf::getAbbreviation(uint64_t code, uint64_t offset) const { // Linear search in the .debug_abbrev section, starting at offset @@ -590,7 +862,7 @@ Dwarf::DIEAbbreviation Dwarf::getAbbreviation(uint64_t code, uint64_t offset) co SAFE_CHECK(false, "could not find abbreviation code"); } -Dwarf::AttributeValue Dwarf::readAttributeValue(std::string_view & sp, uint64_t form, bool is64Bit) const +Dwarf::AttributeValue Dwarf::readAttributeValue(std::string_view & sp, uint64_t form, bool is64_bit) const { switch (form) { @@ -628,26 +900,18 @@ Dwarf::AttributeValue Dwarf::readAttributeValue(std::string_view & sp, uint64_t return uint64_t(1); case DW_FORM_sec_offset: [[fallthrough]]; case DW_FORM_ref_addr: - return readOffset(sp, is64Bit); + return readOffset(sp, is64_bit); case DW_FORM_string: return readNullTerminated(sp); case DW_FORM_strp: - return getStringFromStringSection(readOffset(sp, is64Bit)); + return getStringFromStringSection(str_, readOffset(sp, is64_bit)); case DW_FORM_indirect: // form is explicitly specified - return readAttributeValue(sp, readULEB(sp), is64Bit); + return readAttributeValue(sp, readULEB(sp), is64_bit); default: SAFE_CHECK(false, "invalid attribute form"); } } -std::string_view Dwarf::getStringFromStringSection(uint64_t offset) const -{ - SAFE_CHECK(offset < strings_.size(), "invalid strp offset"); - std::string_view sp(strings_); - sp.remove_prefix(offset); - return readNullTerminated(sp); -} - /** * Find @address in .debug_aranges and return the offset in * .debug_info for compilation unit to which this address belongs. @@ -724,7 +988,7 @@ bool Dwarf::findLocation( // Partial compilation unit (DW_TAG_partial_unit) is not supported. SAFE_CHECK(die.abbr.tag == DW_TAG_compile_unit, "expecting compile unit entry"); - // Read attributes, extracting the few we care about + // Offset in .debug_line for the line number VM program for this CU std::optional line_offset = 0; std::string_view compilation_directory; std::optional main_file_name; @@ -772,7 +1036,7 @@ bool Dwarf::findLocation( std::string_view line_section(line_); line_section.remove_prefix(*line_offset); - LineNumberVM line_vm(line_section, compilation_directory); + LineNumberVM line_vm(line_section, compilation_directory, str_, line_str_); // Execute line number VM program to find file and line info.has_file_and_line = line_vm.findAddress(address, info.file, info.line); @@ -863,8 +1127,11 @@ bool Dwarf::findLocation( return info.has_file_and_line; } -void Dwarf::findSubProgramDieForAddress( - const CompilationUnit & cu, const Die & die, uint64_t address, std::optional base_addr_cu, Die & subprogram) const +void Dwarf::findSubProgramDieForAddress(const CompilationUnit & cu, + const Die & die, + uint64_t address, + std::optional base_addr_cu, + Die & subprogram) const { forEachChild(cu, die, [&](const Die & child_die) { @@ -885,9 +1152,14 @@ void Dwarf::findSubProgramDieForAddress( low_pc = std::get(attr.attr_value); break; case DW_AT_high_pc: - // Value of DW_AT_high_pc attribute can be an address - // (DW_FORM_addr) or an offset (DW_FORM_data). - is_high_pc_addr = (attr.spec.form == DW_FORM_addr); + // The value of the DW_AT_high_pc attribute can be + // an address (DW_FORM_addr*) or an offset (DW_FORM_data*). + is_high_pc_addr = attr.spec.form == DW_FORM_addr || // + attr.spec.form == DW_FORM_addrx || // + attr.spec.form == DW_FORM_addrx1 || // + attr.spec.form == DW_FORM_addrx2 || // + attr.spec.form == DW_FORM_addrx3 || // + attr.spec.form == DW_FORM_addrx4; high_pc = std::get(attr.attr_value); break; } @@ -896,7 +1168,7 @@ void Dwarf::findSubProgramDieForAddress( }); bool pc_match = low_pc && high_pc && is_high_pc_addr && address >= *low_pc && (address < (*is_high_pc_addr ? *high_pc : *low_pc + *high_pc)); - bool range_match = range_offset && isAddrInRangeList(address, base_addr_cu, range_offset.value(), cu.addr_size); + bool range_match = range_offset && isAddrInRangeList(cu, address, base_addr_cu, range_offset.value(), cu.addr_size); if (pc_match || range_match) { subprogram = child_die; @@ -971,9 +1243,14 @@ void Dwarf::findInlinedSubroutineDieForAddress( low_pc = std::get(attr.attr_value); break; case DW_AT_high_pc: - // Value of DW_AT_high_pc attribute can be an address - // (DW_FORM_addr) or an offset (DW_FORM_data). - is_high_pc_addr = (attr.spec.form == DW_FORM_addr); + // The value of the DW_AT_high_pc attribute can be + // an address (DW_FORM_addr*) or an offset (DW_FORM_data*). + is_high_pc_addr = attr.spec.form == DW_FORM_addr || // + attr.spec.form == DW_FORM_addrx || // + attr.spec.form == DW_FORM_addrx1 || // + attr.spec.form == DW_FORM_addrx2 || // + attr.spec.form == DW_FORM_addrx3 || // + attr.spec.form == DW_FORM_addrx4; high_pc = std::get(attr.attr_value); break; case DW_AT_abstract_origin: @@ -1005,7 +1282,7 @@ void Dwarf::findInlinedSubroutineDieForAddress( // TODO: Support relocated address which requires lookup in relocation map. bool pc_match = low_pc && high_pc && is_high_pc_addr && address >= *low_pc && (address < (*is_high_pc_addr ? *high_pc : *low_pc + *high_pc)); - bool range_match = range_offset && isAddrInRangeList(address, base_addr_cu, range_offset.value(), cu.addr_size); + bool range_match = range_offset && isAddrInRangeList(cu, address, base_addr_cu, range_offset.value(), cu.addr_size); if (!pc_match && !range_match) { // Address doesn't match. Keep searching other children. @@ -1107,7 +1384,7 @@ void Dwarf::findInlinedSubroutineDieForAddress( // Not applicable for DW_AT_abstract_origin. location.name = (*abstract_origin_ref_type != DW_FORM_ref_addr) ? get_function_name(cu, cu.offset + *abstract_origin) - : get_function_name(findCompilationUnit(info_, *abstract_origin), *abstract_origin); + : get_function_name(findCompilationUnit(*abstract_origin), *abstract_origin); /// FIXME: see comment above if (die_for_inline_broken) @@ -1144,7 +1421,11 @@ bool Dwarf::findAddress( if (findDebugInfoOffset(address, aranges_, offset)) { // Read compilation unit header from .debug_info - auto unit = getCompilationUnit(info_, offset); + auto unit = getCompilationUnit(offset); + if (unit.unit_type != DW_UT_compile && unit.unit_type != DW_UT_skeleton) + { + return false; + } findLocation(address, mode, unit, locationInfo, inline_frames); return locationInfo.has_file_and_line; } @@ -1168,84 +1449,160 @@ bool Dwarf::findAddress( uint64_t offset = 0; while (offset < info_.size() && !locationInfo.has_file_and_line) { - auto unit = getCompilationUnit(info_, offset); + auto unit = getCompilationUnit(offset); offset += unit.size; + if (unit.unit_type != DW_UT_compile && unit.unit_type != DW_UT_skeleton) + { + continue; + } findLocation(address, mode, unit, locationInfo, inline_frames); } return locationInfo.has_file_and_line; } -bool Dwarf::isAddrInRangeList(uint64_t address, std::optional base_addr, size_t offset, uint8_t addr_size) const +bool Dwarf::isAddrInRangeList(const CompilationUnit & cu, + uint64_t address, + std::optional base_addr, + size_t offset, + uint8_t addr_size) const { SAFE_CHECK(addr_size == 4 || addr_size == 8, "wrong address size"); - if (ranges_.empty()) + if (cu.version <= 4 && !ranges_.empty()) { - return false; - } + const bool is64_bit_addr = addr_size == 8; + std::string_view sp = ranges_; + sp.remove_prefix(offset); + const uint64_t max_addr = is64_bit_addr ? std::numeric_limits::max() : std::numeric_limits::max(); + while (!sp.empty()) + { + uint64_t begin = readOffset(sp, is64_bit_addr); + uint64_t end = readOffset(sp, is64_bit_addr); + // The range list entry is a base address selection entry. + if (begin == max_addr) + { + base_addr = end; + continue; + } + // The range list entry is an end of list entry. + if (begin == 0 && end == 0) + { + break; + } - const bool is_64bit_addr = addr_size == 8; - std::string_view sp = ranges_; - sp.remove_prefix(offset); - const uint64_t max_addr = is_64bit_addr ? std::numeric_limits::max() : std::numeric_limits::max(); - while (!sp.empty()) - { - uint64_t begin = readOffset(sp, is_64bit_addr); - uint64_t end = readOffset(sp, is_64bit_addr); - // The range list entry is a base address selection entry. - if (begin == max_addr) - { - base_addr = end; - continue; - } - // The range list entry is an end of list entry. - if (begin == 0 && end == 0) - { - break; - } - // Check if the given address falls in the range list entry. - // 2.17.3 Non-Contiguous Address Ranges - // The applicable base address of a range list entry is determined by the - // closest preceding base address selection entry (see below) in the same - // range list. If there is no such selection entry, then the applicable base - // address defaults to the base address of the compilation unit. - if (base_addr && address >= begin + *base_addr && address < end + *base_addr) - { - return true; + // Check if the given address falls in the range list entry. + // 2.17.3 Non-Contiguous Address Ranges + // The applicable base address of a range list entry is determined by the + // closest preceding base address selection entry (see below) in the same + // range list. If there is no such selection entry, then the applicable + // base address defaults to the base address of the compilation unit. + if (base_addr && address >= begin + *base_addr && address < end + *base_addr) + { + return true; + } } } + if (cu.version == 5 && !rnglists_.empty() && cu.addr_base.has_value()) + { + auto rnglists = rnglists_; + rnglists.remove_prefix(offset); + + while (!rnglists.empty()) + { + auto kind = read(rnglists); + switch (kind) + { + case DW_RLE_end_of_list: + return false; + case DW_RLE_base_addressx: { + auto index = readULEB(rnglists); + auto sp = addr_.substr(*cu.addr_base + index * sizeof(uint64_t)); + base_addr = read(sp); + } + break; + + case DW_RLE_startx_endx: { + auto index_start = readULEB(rnglists); + auto index_end = readULEB(rnglists); + auto sp_start = addr_.substr(*cu.addr_base + index_start * sizeof(uint64_t)); + auto start = read(sp_start); + + auto sp_end = addr_.substr(*cu.addr_base + index_end * sizeof(uint64_t)); + auto end = read(sp_end); + if (address >= start && address < end) + { + return true; + } + } + break; + + case DW_RLE_startx_length: { + auto index_start = readULEB(rnglists); + auto length = readULEB(rnglists); + auto sp_start = addr_.substr(*cu.addr_base + index_start * sizeof(uint64_t)); + auto start = read(sp_start); + + auto sp_end = addr_.substr(*cu.addr_base + index_start * sizeof(uint64_t) + length); + auto end = read(sp_end); + if (start != end && address >= start && address < end) + { + return true; + } + } + break; + + case DW_RLE_offset_pair: { + auto offset_start = readULEB(rnglists); + auto offset_end = readULEB(rnglists); + if (base_addr && address >= (*base_addr + offset_start) && address < (*base_addr + offset_end)) + { + return true; + } + } + break; + + case DW_RLE_base_address: + base_addr = read(rnglists); + break; + + case DW_RLE_start_end: { + uint64_t start = read(rnglists); + uint64_t end = read(rnglists); + if (address >= start && address < end) + { + return true; + } + } + break; + + case DW_RLE_start_length: { + uint64_t start = read(rnglists); + uint64_t end = start + readULEB(rnglists); + if (address >= start && address < end) + { + return true; + } + } + break; + + default: + SAFE_CHECK(false, "Unexpected debug_rnglists entry kind"); + } + } + } return false; } -// static -Dwarf::CompilationUnit Dwarf::findCompilationUnit(std::string_view info, uint64_t targetOffset) -{ - SAFE_CHECK(targetOffset < info.size(), "unexpected target address"); - uint64_t offset = 0; - while (offset < info.size()) - { - std::string_view chunk(info); - chunk.remove_prefix(offset); - auto initial_length = read(chunk); - auto is_64bit = (initial_length == uint32_t(-1)); - auto size = is_64bit ? read(chunk) : initial_length; - SAFE_CHECK(size <= chunk.size(), "invalid chunk size"); - size += is_64bit ? 12 : 4; - - if (offset + size > targetOffset) - { - break; - } - offset += size; - } - return getCompilationUnit(info, offset); -} - - -Dwarf::LineNumberVM::LineNumberVM(std::string_view data, std::string_view compilationDirectory) +Dwarf::LineNumberVM::LineNumberVM( + std::string_view data, + std::string_view compilationDirectory, + std::string_view debugStr, + std::string_view debugLineStr) : compilationDirectory_(compilationDirectory) + , debugStr_(debugStr) + , debugLineStr_(debugLineStr) { Section section(data); SAFE_CHECK(section.next(data_), "invalid line number VM"); @@ -1269,17 +1626,154 @@ void Dwarf::LineNumberVM::reset() discriminator_ = 0; } +struct LineNumberAttribute +{ + uint64_t content_type_code; + uint64_t form_code; + std::variant attr_value; +}; + +LineNumberAttribute readLineNumberAttribute( + bool is64_bit, std::string_view & format, std::string_view & entries, std::string_view debugStr, std::string_view debugLineStr) +{ + uint64_t content_type_code = readULEB(format); + uint64_t form_code = readULEB(format); + std::variant attr_value; + + switch (content_type_code) + { + case DW_LNCT_path: { + switch (form_code) + { + case DW_FORM_string: + attr_value = readNullTerminated(entries); + break; + case DW_FORM_line_strp: { + auto off = readOffset(entries, is64_bit); + attr_value = getStringFromStringSection(debugLineStr, off); + } + break; + case DW_FORM_strp: + attr_value = getStringFromStringSection(debugStr, readOffset(entries, is64_bit)); + break; + case DW_FORM_strp_sup: + SAFE_CHECK(false, "Unexpected DW_FORM_strp_sup"); + break; + default: + SAFE_CHECK(false, "Unexpected form for DW_LNCT_path"); + break; + } + } + break; + + case DW_LNCT_directory_index: { + switch (form_code) + { + case DW_FORM_data1: + attr_value = read(entries); + break; + case DW_FORM_data2: + attr_value = read(entries); + break; + case DW_FORM_udata: + attr_value = readULEB(entries); + break; + default: + SAFE_CHECK(false, "Unexpected form for DW_LNCT_directory_index"); + break; + } + } + break; + + case DW_LNCT_timestamp: { + switch (form_code) + { + case DW_FORM_udata: + attr_value = readULEB(entries); + break; + case DW_FORM_data4: + attr_value = read(entries); + break; + case DW_FORM_data8: + attr_value = read(entries); + break; + case DW_FORM_block: + attr_value = readBytes(entries, readULEB(entries)); + break; + default: + SAFE_CHECK(false, "Unexpected form for DW_LNCT_timestamp"); + } + } + break; + + case DW_LNCT_size: { + switch (form_code) + { + case DW_FORM_udata: + attr_value = readULEB(entries); + break; + case DW_FORM_data1: + attr_value = read(entries); + break; + case DW_FORM_data2: + attr_value = read(entries); + break; + case DW_FORM_data4: + attr_value = read(entries); + break; + case DW_FORM_data8: + attr_value = read(entries); + break; + default: + SAFE_CHECK(false, "Unexpected form for DW_LNCT_size"); + break; + } + } + break; + + case DW_LNCT_MD5: { + switch (form_code) + { + case DW_FORM_data16: + attr_value = readBytes(entries, 16); + break; + default: + SAFE_CHECK(false, "Unexpected form for DW_LNCT_MD5"); + break; + } + } + break; + + default: + // TODO: skip over vendor data as specified by the form instead. + SAFE_CHECK(false, "Unexpected vendor content type code"); + break; + } + return { + .content_type_code = content_type_code, + .form_code = form_code, + .attr_value = attr_value, + }; +} + void Dwarf::LineNumberVM::init() { version_ = read(data_); - SAFE_CHECK(version_ >= 2 && version_ <= 4, "invalid version in line number VM"); + SAFE_CHECK(version_ >= 2 && version_ <= 5, "invalid version in line number VM: {}", version_); + if (version_ == 5) + { + auto address_size = read(data_); + SAFE_CHECK(address_size == sizeof(uintptr_t), "Unexpected Line Number Table address_size"); + auto segment_selector_size = read(data_); + SAFE_CHECK(segment_selector_size == 0, "Segments not supported"); + } uint64_t header_length = readOffset(data_, is64Bit_); SAFE_CHECK(header_length <= data_.size(), "invalid line number VM header length"); std::string_view header(data_.data(), header_length); data_ = std::string_view(header.end(), data_.end() - header.end()); minLength_ = read(header); - if (version_ == 4) + if (version_ >= 4) { // Version 2 and 3 records don't have this uint8_t max_ops_per_instruction = read(header); SAFE_CHECK(max_ops_per_instruction == 1, "VLIW not supported"); @@ -1292,26 +1786,75 @@ void Dwarf::LineNumberVM::init() standardOpcodeLengths_ = reinterpret_cast(header.data()); //-V506 header.remove_prefix(opcodeBase_ - 1); - // We don't want to use heap, so we don't keep an unbounded amount of state. - // We'll just skip over include directories and file names here, and - // we'll loop again when we actually need to retrieve one. - std::string_view sp; - const char * tmp = header.data(); - includeDirectoryCount_ = 0; - while (!(sp = readNullTerminated(header)).empty()) + if (version_ <= 4) { - ++includeDirectoryCount_; - } - includeDirectories_ = std::string_view(tmp, header.data() - tmp); + // We don't want to use heap, so we don't keep an unbounded amount of state. + // We'll just skip over include directories and file names here, and + // we'll loop again when we actually need to retrieve one. + std::string_view sp; + const char * tmp = header.data(); + v4_.includeDirectoryCount = 0; + while (!(sp = readNullTerminated(header)).empty()) + { + ++v4_.includeDirectoryCount; + } + v4_.includeDirectories = {tmp, header.data()}; - tmp = header.data(); - FileName fn; - fileNameCount_ = 0; - while (readFileName(header, fn)) - { - ++fileNameCount_; + tmp = header.data(); + FileName fn; + v4_.fileNameCount = 0; + while (readFileName(header, fn)) + { + ++v4_.fileNameCount; + } + v4_.fileNames = {tmp, header.data()}; + } + else if (version_ == 5) + { + v5_.directoryEntryFormatCount = read(header); + const char * tmp = header.data(); + for (uint8_t i = 0; i < v5_.directoryEntryFormatCount; i++) + { + // A sequence of directory entry format descriptions. Each description + // consists of a pair of ULEB128 values: + readULEB(header); // A content type code + readULEB(header); // A form code using the attribute form codes + } + v5_.directoryEntryFormat = {tmp, header.data()}; + v5_.directoriesCount = readULEB(header); + tmp = header.data(); + for (uint64_t i = 0; i < v5_.directoriesCount; i++) + { + std::string_view format = v5_.directoryEntryFormat; + for (uint8_t f = 0; f < v5_.directoryEntryFormatCount; f++) + { + readLineNumberAttribute(is64Bit_, format, header, debugStr_, debugLineStr_); + } + } + v5_.directories = {tmp, header.data()}; + + v5_.fileNameEntryFormatCount = read(header); + tmp = header.data(); + for (uint8_t i = 0; i < v5_.fileNameEntryFormatCount; i++) + { + // A sequence of file entry format descriptions. Each description + // consists of a pair of ULEB128 values: + readULEB(header); // A content type code + readULEB(header); // A form code using the attribute form codes + } + v5_.fileNameEntryFormat = {tmp, header.data()}; + v5_.fileNamesCount = readULEB(header); + tmp = header.data(); + for (uint64_t i = 0; i < v5_.fileNamesCount; i++) + { + std::string_view format = v5_.fileNameEntryFormat; + for (uint8_t f = 0; f < v5_.fileNameEntryFormatCount; f++) + { + readLineNumberAttribute(is64Bit_, format, header, debugStr_, debugLineStr_); + } + } + v5_.fileNames = {tmp, header.data()}; } - fileNames_ = std::string_view(tmp, header.data() - tmp); } bool Dwarf::LineNumberVM::next(std::string_view & program) @@ -1327,54 +1870,110 @@ bool Dwarf::LineNumberVM::next(std::string_view & program) Dwarf::LineNumberVM::FileName Dwarf::LineNumberVM::getFileName(uint64_t index) const { - SAFE_CHECK(index != 0, "invalid file index 0"); - - FileName fn; - if (index <= fileNameCount_) + if (version_ <= 4) { - std::string_view file_names = fileNames_; + SAFE_CHECK(index != 0, "invalid file index 0"); + FileName fn; + if (index <= v4_.fileNameCount) + { + std::string_view file_names = v4_.fileNames; + for (; index; --index) + { + if (!readFileName(file_names, fn)) + { + abort(); + } + } + return fn; + } + + index -= v4_.fileNameCount; + + std::string_view program = data_; for (; index; --index) { - if (!readFileName(file_names, fn)) + SAFE_CHECK(nextDefineFile(program, fn), "invalid file index"); + } + + return fn; + } + else + { + FileName fn; + SAFE_CHECK(index < v5_.fileNamesCount, "invalid file index"); + std::string_view file_names = v5_.fileNames; + for (uint64_t i = 0; i < v5_.fileNamesCount; i++) + { + std::string_view format = v5_.fileNameEntryFormat; + for (uint8_t f = 0; f < v5_.fileNameEntryFormatCount; f++) { - abort(); + auto attr = readLineNumberAttribute(is64Bit_, format, file_names, debugStr_, debugLineStr_); + if (i == index) + { + switch (attr.content_type_code) + { + case DW_LNCT_path: + fn.relativeName = std::get(attr.attr_value); + break; + case DW_LNCT_directory_index: + fn.directoryIndex = std::get(attr.attr_value); + break; + } + } } } return fn; } - - index -= fileNameCount_; - - std::string_view program = data_; - for (; index; --index) - { - SAFE_CHECK(nextDefineFile(program, fn), "invalid file index"); - } - - return fn; } std::string_view Dwarf::LineNumberVM::getIncludeDirectory(uint64_t index) const { - if (index == 0) + if (version_ <= 4) { - return std::string_view(); - } - - SAFE_CHECK(index <= includeDirectoryCount_, "invalid include directory"); - - std::string_view include_directories = includeDirectories_; - std::string_view dir; - for (; index; --index) - { - dir = readNullTerminated(include_directories); - if (dir.empty()) + if (index == 0) { - abort(); // BUG + // In DWARF <= 4 the current directory is not represented in the + // directories field and a directory index of 0 implicitly referred to + // that directory as found in the DW_AT_comp_dir attribute of the + // compilation unit debugging information entry. + return {}; } - } - return dir; + SAFE_CHECK(index <= v4_.includeDirectoryCount, "invalid include directory"); + + std::string_view include_directories = v4_.includeDirectories; + std::string_view dir; + for (; index; --index) + { + dir = readNullTerminated(include_directories); + if (dir.empty()) + { + abort(); // BUG + } + } + + return dir; + } + else + { + SAFE_CHECK(index < v5_.directoriesCount, "invalid file index"); + std::string_view directories = v5_.directories; + for (uint64_t i = 0; i < v5_.directoriesCount; i++) + { + std::string_view format = v5_.directoryEntryFormat; + for (uint8_t f = 0; f < v5_.directoryEntryFormatCount; f++) + { + auto attr = readLineNumberAttribute(is64Bit_, format, directories, debugStr_, debugLineStr_); + if (i == index && attr.content_type_code == DW_LNCT_path) + { + return std::get(attr.attr_value); + } + } + } + // This could only happen if DWARF5's directory_entry_format doesn't contain + // a DW_LNCT_path. Highly unlikely, but we shouldn't crash. + return std::string_view(""); + } } bool Dwarf::LineNumberVM::readFileName(std::string_view & program, FileName & fn) @@ -1422,6 +2021,7 @@ bool Dwarf::LineNumberVM::nextDefineFile(std::string_view & program, FileName & if (opcode == DW_LNE_define_file) { + SAFE_CHECK(version_ < 5, "DW_LNE_define_file deprecated in DWARF5"); SAFE_CHECK(readFileName(program, fn), "invalid empty file in DW_LNE_define_file"); return true; } @@ -1535,6 +2135,7 @@ Dwarf::LineNumberVM::StepResult Dwarf::LineNumberVM::step(std::string_view & pro address_ = read(program); return CONTINUE; case DW_LNE_define_file: + SAFE_CHECK(version_ < 5, "DW_LNE_define_file deprecated in DWARF5"); // We can't process DW_LNE_define_file here, as it would require us to // use unbounded amounts of state (ie. use the heap). We'll do a second // pass (using nextDefineFile()) if necessary. @@ -1549,6 +2150,16 @@ Dwarf::LineNumberVM::StepResult Dwarf::LineNumberVM::step(std::string_view & pro return CONTINUE; } +Dwarf::Path Dwarf::LineNumberVM::getFullFileName(uint64_t index) const +{ + auto fn = getFileName(index); + // DWARF <= 4: the current dir is not represented in the CU's Line Number + // Program Header and relies on the CU's DW_AT_comp_dir. + // DWARF 5: the current directory is explicitly present. + const std::string_view base_dir = version_ == 5 ? "" : compilationDirectory_; + return Path(base_dir, getIncludeDirectory(fn.directoryIndex), fn.relativeName); +} + bool Dwarf::LineNumberVM::findAddress(uintptr_t target, Path & file, uint64_t & line) { std::string_view program = data_; @@ -1588,12 +2199,18 @@ bool Dwarf::LineNumberVM::findAddress(uintptr_t target, Path & file, uint64_t & // Found it! Note that ">" is indeed correct (not ">="), as each // sequence is guaranteed to have one entry past-the-end (emitted by // DW_LNE_end_sequence) - if (prev_file == 0) + // + // NOTE: In DWARF <= 4 the file register is non-zero. + // See DWARF 4: 6.2.4 The Line Number Program Header + // "The line number program assigns numbers to each of the file + // entries in order, beginning with 1, and uses those numbers instead + // of file names in the file register." + // DWARF 5 has a different include directory/file header and 0 is valid. + if (version_ <= 4 && prev_file == 0) { return false; } - auto fn = getFileName(prev_file); - file = Path(compilationDirectory_, getIncludeDirectory(fn.directoryIndex), fn.relativeName); + file = getFullFileName(prev_file); line = prev_line; return true; } diff --git a/src/Common/Dwarf.h b/src/Common/Dwarf.h index 6e3d3e74e81..09178c66d47 100644 --- a/src/Common/Dwarf.h +++ b/src/Common/Dwarf.h @@ -19,6 +19,7 @@ */ /** This file was edited for ClickHouse. + * Original is from folly library. */ #include @@ -113,8 +114,8 @@ public: // seems as the same path can be represented in multiple ways private: std::string_view baseDir_; /// NOLINT - std::string_view subDir_; /// NOLINT - std::string_view file_; /// NOLINT + std::string_view subDir_; /// NOLINT + std::string_view file_; /// NOLINT }; // Indicates inline function `name` is called at `line@file`. @@ -171,8 +172,6 @@ public: private: static bool findDebugInfoOffset(uintptr_t address, std::string_view aranges, uint64_t & offset); - void init(); - std::shared_ptr elf_; /// NOLINT // DWARF section made up of chunks, each prefixed with a length header. @@ -228,6 +227,7 @@ private: { uint64_t name = 0; uint64_t form = 0; + int64_t implicitConst = 0; // only set when form=DW_FORM_implicit_const explicit operator bool() const { return name != 0 || form != 0; } }; @@ -239,25 +239,43 @@ private: std::variant attr_value; }; + enum + { + DW_UT_compile = 0x01, + DW_UT_skeleton = 0x04, + }; + struct CompilationUnit { - bool is64Bit; /// NOLINT - uint8_t version; - uint8_t addr_size; + bool is64Bit = false; /// NOLINT + uint8_t version = 0; + uint8_t unit_type = DW_UT_compile; // DW_UT_compile or DW_UT_skeleton + uint8_t addr_size = 0; // Offset in .debug_info of this compilation unit. - uint32_t offset; - uint32_t size; + uint32_t offset = 0; + uint32_t size = 0; // Offset in .debug_info for the first DIE in this compilation unit. - uint32_t first_die; - uint64_t abbrev_offset; + uint32_t first_die = 0; + uint64_t abbrev_offset = 0; + + // The beginning of the CU's contribution to .debug_addr + std::optional addr_base; // DW_AT_addr_base (DWARF 5) + // The beginning of the offsets table (immediately following the + // header) of the CU's contribution to .debug_loclists + std::optional loclists_base; // DW_AT_loclists_base (DWARF 5) + // The beginning of the offsets table (immediately following the + // header) of the CU's contribution to .debug_rnglists + std::optional rnglists_base; // DW_AT_rnglists_base (DWARF 5) + // Points to the first string offset of the compilation unit’s + // contribution to the .debug_str_offsets (or .debug_str_offsets.dwo) section. + std::optional str_offsets_base; // DW_AT_str_offsets_base (DWARF 5) + // Only the CompilationUnit that contains the caller functions needs this cache. // Indexed by (abbr.code - 1) if (abbr.code - 1) < abbrCache.size(); std::vector abbr_cache; }; - static CompilationUnit getCompilationUnit(std::string_view info, uint64_t offset); - - /** cu must exist during the life cycle of created detail::Die. */ + /** cu must exist during the life cycle of created Die. */ Die getDieAtOffset(const CompilationUnit & cu, uint64_t offset) const; bool findLocation( @@ -278,16 +296,16 @@ private: class LineNumberVM { public: - LineNumberVM(std::string_view data, std::string_view compilationDirectory); + LineNumberVM( + std::string_view data, + std::string_view compilationDirectory, + std::string_view debugStr, + std::string_view debugLineStr); bool findAddress(uintptr_t target, Path & file, uint64_t & line); /** Gets full file name at given index including directory. */ - Path getFullFileName(uint64_t index) const - { - auto fn = getFileName(index); - return Path({}, getIncludeDirectory(fn.directoryIndex), fn.relativeName); - } + Path getFullFileName(uint64_t index) const; private: void init(); @@ -327,24 +345,42 @@ private: bool nextDefineFile(std::string_view & program, FileName & fn) const; // Initialization - bool is64Bit_; /// NOLINT - std::string_view data_; /// NOLINT - std::string_view compilationDirectory_; /// NOLINT + bool is64Bit_; /// NOLINT + std::string_view data_; /// NOLINT + std::string_view compilationDirectory_; /// NOLINT + std::string_view debugStr_; // needed for DWARF 5 /// NOLINT + std::string_view debugLineStr_; // DWARF 5 /// NOLINT // Header - uint16_t version_; /// NOLINT - uint8_t minLength_; /// NOLINT + uint16_t version_; /// NOLINT + uint8_t minLength_; /// NOLINT bool defaultIsStmt_; /// NOLINT - int8_t lineBase_; /// NOLINT - uint8_t lineRange_; /// NOLINT + int8_t lineBase_; /// NOLINT + uint8_t lineRange_; /// NOLINT uint8_t opcodeBase_; /// NOLINT const uint8_t * standardOpcodeLengths_; /// NOLINT - std::string_view includeDirectories_; /// NOLINT - size_t includeDirectoryCount_; /// NOLINT + // 6.2.4 The Line Number Program Header. + struct + { + size_t includeDirectoryCount; + std::string_view includeDirectories; + size_t fileNameCount; + std::string_view fileNames; + } v4_; - std::string_view fileNames_; /// NOLINT - size_t fileNameCount_; /// NOLINT + struct + { + uint8_t directoryEntryFormatCount; + std::string_view directoryEntryFormat; + uint64_t directoriesCount; + std::string_view directories; + + uint8_t fileNameEntryFormatCount; + std::string_view fileNameEntryFormat; + uint64_t fileNamesCount; + std::string_view fileNames; + } v5_; // State machine registers uint64_t address_; /// NOLINT @@ -397,20 +433,26 @@ private: */ size_t forEachAttribute(const CompilationUnit & cu, const Die & die, std::function f) const; - Attribute readAttribute(const Die & die, AttributeSpec spec, std::string_view & info) const; + Attribute readAttribute( + const CompilationUnit & cu, + const Die & die, + AttributeSpec spec, + std::string_view & info) const; // Read one attribute pair, remove_prefix sp; returns <0, 0> at end. static AttributeSpec readAttributeSpec(std::string_view & sp); // Read one attribute value, remove_prefix sp using AttributeValue = std::variant; - AttributeValue readAttributeValue(std::string_view & sp, uint64_t form, bool is64Bit) const; + AttributeValue readAttributeValue(std::string_view & sp, uint64_t form, bool is64_bit) const; // Get an ELF section by name, return true if found - bool getSection(const char * name, std::string_view * section) const; + std::string_view getSection(const char * name) const; + + CompilationUnit getCompilationUnit(uint64_t offset) const; + // Finds the Compilation Unit starting at offset. + CompilationUnit findCompilationUnit(uint64_t targetOffset) const; - // Get a string from the .debug_str section - std::string_view getStringFromStringSection(uint64_t offset) const; template std::optional getAttribute(const CompilationUnit & cu, const Die & die, uint64_t attr_name) const @@ -429,17 +471,24 @@ private: } // Check if the given address is in the range list at the given offset in .debug_ranges. - bool isAddrInRangeList(uint64_t address, std::optional base_addr, size_t offset, uint8_t addr_size) const; + bool isAddrInRangeList( + const CompilationUnit & cu, + uint64_t address, + std::optional base_addr, + size_t offset, + uint8_t addr_size) const; - // Finds the Compilation Unit starting at offset. - static CompilationUnit findCompilationUnit(std::string_view info, uint64_t targetOffset); - - std::string_view info_; // .debug_info /// NOLINT - std::string_view abbrev_; // .debug_abbrev /// NOLINT - std::string_view aranges_; // .debug_aranges /// NOLINT - std::string_view line_; // .debug_line /// NOLINT - std::string_view strings_; // .debug_str /// NOLINT - std::string_view ranges_; // .debug_ranges /// NOLINT + std::string_view abbrev_; // .debug_abbrev /// NOLINT + std::string_view addr_; // .debug_addr (DWARF 5) /// NOLINT + std::string_view aranges_; // .debug_aranges /// NOLINT + std::string_view info_; // .debug_info /// NOLINT + std::string_view line_; // .debug_line /// NOLINT + std::string_view line_str_; // .debug_line_str (DWARF 5) /// NOLINT + std::string_view loclists_; // .debug_loclists (DWARF 5) /// NOLINT + std::string_view ranges_; // .debug_ranges /// NOLINT + std::string_view rnglists_; // .debug_rnglists (DWARF 5) /// NOLINT + std::string_view str_; // .debug_str /// NOLINT + std::string_view str_offsets_; // .debug_str_offsets (DWARF 5) /// NOLINT }; } diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index 8f46cea25a0..e80ad5c141a 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -636,6 +636,7 @@ M(665, CANNOT_CONNECT_NATS) \ M(666, CANNOT_USE_CACHE) \ M(667, NOT_INITIALIZED) \ + M(668, INVALID_STATE) \ \ M(999, KEEPER_EXCEPTION) \ M(1000, POCO_EXCEPTION) \ diff --git a/src/Common/Exception.cpp b/src/Common/Exception.cpp index 3645ac5594f..931f06fdb51 100644 --- a/src/Common/Exception.cpp +++ b/src/Common/Exception.cpp @@ -176,10 +176,10 @@ static void tryLogCurrentExceptionImpl(Poco::Logger * logger, const std::string void tryLogCurrentException(const char * log_name, const std::string & start_of_message) { - /// Under high memory pressure, any new allocation will definitelly lead - /// to MEMORY_LIMIT_EXCEEDED exception. + /// Under high memory pressure, new allocations throw a + /// MEMORY_LIMIT_EXCEEDED exception. /// - /// And in this case the exception will not be logged, so let's block the + /// In this case the exception will not be logged, so let's block the /// MemoryTracker until the exception will be logged. LockMemoryExceptionInThread lock_memory_tracker(VariableContext::Global); @@ -189,8 +189,8 @@ void tryLogCurrentException(const char * log_name, const std::string & start_of_ void tryLogCurrentException(Poco::Logger * logger, const std::string & start_of_message) { - /// Under high memory pressure, any new allocation will definitelly lead - /// to MEMORY_LIMIT_EXCEEDED exception. + /// Under high memory pressure, new allocations throw a + /// MEMORY_LIMIT_EXCEEDED exception. /// /// And in this case the exception will not be logged, so let's block the /// MemoryTracker until the exception will be logged. diff --git a/src/Common/ExternalModelInfo.h b/src/Common/ExternalModelInfo.h new file mode 100644 index 00000000000..378e4984af6 --- /dev/null +++ b/src/Common/ExternalModelInfo.h @@ -0,0 +1,20 @@ +#pragma once + +#include +#include + +namespace DB +{ + +/// Details about external machine learning model, used by clickhouse-server and clickhouse-library-bridge +struct ExternalModelInfo +{ + String model_path; + String model_type; + std::chrono::system_clock::time_point loading_start_time; /// serialized as std::time_t + std::chrono::milliseconds loading_duration; /// serialized as UInt64 +}; + +using ExternalModelInfos = std::vector; + +} diff --git a/src/Common/FieldVisitorConvertToNumber.h b/src/Common/FieldVisitorConvertToNumber.h index 92da0f89844..466d312406e 100644 --- a/src/Common/FieldVisitorConvertToNumber.h +++ b/src/Common/FieldVisitorConvertToNumber.h @@ -94,21 +94,7 @@ public: T operator() (const DecimalField & x) const { if constexpr (std::is_floating_point_v) - return x.getValue(). template convertTo() / x.getScaleMultiplier(). template convertTo(); - else if constexpr (std::is_same_v) - { - if constexpr (sizeof(U) < 16) - { - return UInt128(0, (x.getValue() / x.getScaleMultiplier()).value); - } - else if constexpr (sizeof(U) == 16) - { - auto tmp = (x.getValue() / x.getScaleMultiplier()).value; - return UInt128(tmp >> 64, UInt64(tmp)); - } - else - throw Exception("No conversion to old UInt128 from " + demangle(typeid(U).name()), ErrorCodes::NOT_IMPLEMENTED); - } + return x.getValue().template convertTo() / x.getScaleMultiplier().template convertTo(); else return (x.getValue() / x.getScaleMultiplier()). template convertTo(); } @@ -134,4 +120,3 @@ public: }; } - diff --git a/src/Common/FieldVisitorSum.cpp b/src/Common/FieldVisitorSum.cpp index bc996ae2298..2c404c33177 100644 --- a/src/Common/FieldVisitorSum.cpp +++ b/src/Common/FieldVisitorSum.cpp @@ -15,7 +15,7 @@ FieldVisitorSum::FieldVisitorSum(const Field & rhs_) : rhs(rhs_) {} bool FieldVisitorSum::operator() (Int64 & x) const { return this->operator()(reinterpret_cast(x)); } bool FieldVisitorSum::operator() (UInt64 & x) const { - x += rhs.reinterpret(); + x += applyVisitor(FieldVisitorConvertToNumber(), rhs); return x != 0; } diff --git a/src/Common/FieldVisitorSum.h b/src/Common/FieldVisitorSum.h index cd8777e7bfb..c28e2058b05 100644 --- a/src/Common/FieldVisitorSum.h +++ b/src/Common/FieldVisitorSum.h @@ -1,6 +1,7 @@ #pragma once #include +#include namespace DB @@ -41,7 +42,7 @@ public: requires is_big_int_v bool operator() (T & x) const { - x += rhs.reinterpret(); + x += applyVisitor(FieldVisitorConvertToNumber(), rhs); return x != T(0); } }; diff --git a/src/Common/MemoryStatisticsOS.cpp b/src/Common/MemoryStatisticsOS.cpp index 22f8446121f..f2d2ab5fea9 100644 --- a/src/Common/MemoryStatisticsOS.cpp +++ b/src/Common/MemoryStatisticsOS.cpp @@ -135,7 +135,7 @@ MemoryStatisticsOS::Data MemoryStatisticsOS::get() const struct kinfo_proc kp; size_t len = sizeof(struct kinfo_proc); - if (-1 == ::sysctl(mib, 4, &kp, &len, NULL, 0)) + if (-1 == ::sysctl(mib, 4, &kp, &len, nullptr, 0)) throwFromErrno("Cannot sysctl(kern.proc.pid." + std::to_string(self) + ")", ErrorCodes::SYSTEM_ERROR); if (sizeof(struct kinfo_proc) != len) diff --git a/src/Common/OpenTelemetryTraceContext.cpp b/src/Common/OpenTelemetryTraceContext.cpp index 7a1f94926d5..d5c2188ad01 100644 --- a/src/Common/OpenTelemetryTraceContext.cpp +++ b/src/Common/OpenTelemetryTraceContext.cpp @@ -130,16 +130,15 @@ void SpanHolder::finish() noexcept try { auto log = current_thread_trace_context.span_log.lock(); - if (!log) + + /// The log might be disabled, check it before use + if (log) { - // The log might be disabled. - return; + this->finish_time_us + = std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count(); + + log->add(OpenTelemetrySpanLogElement(*this)); } - - this->finish_time_us - = std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count(); - - log->add(OpenTelemetrySpanLogElement(*this)); } catch (...) { diff --git a/src/Common/PODArray.h b/src/Common/PODArray.h index d3232d833ee..0baefad39e2 100644 --- a/src/Common/PODArray.h +++ b/src/Common/PODArray.h @@ -115,7 +115,13 @@ protected: } /// Minimum amount of memory to allocate for num_elements, including padding. - static size_t minimum_memory_for_elements(size_t num_elements) { return byte_size(num_elements) + pad_right + pad_left; } /// NOLINT + static size_t minimum_memory_for_elements(size_t num_elements) + { + size_t amount; + if (__builtin_add_overflow(byte_size(num_elements), pad_left + pad_right, &amount)) + throw Exception("Amount of memory requested to allocate is more than allowed", ErrorCodes::CANNOT_ALLOCATE_MEMORY); + return amount; + } void alloc_for_num_elements(size_t num_elements) /// NOLINT { @@ -225,9 +231,7 @@ public: void clear() { c_end = c_start; } template -#if defined(__clang__) ALWAYS_INLINE /// Better performance in clang build, worse performance in gcc build. -#endif void reserve(size_t n, TAllocatorParams &&... allocator_params) { if (n > capacity()) diff --git a/src/Common/ThreadPool.h b/src/Common/ThreadPool.h index fc5377b3783..76ada9e0d75 100644 --- a/src/Common/ThreadPool.h +++ b/src/Common/ThreadPool.h @@ -264,6 +264,18 @@ protected: } }; +/// Schedule jobs/tasks on global thread pool without implicit passing tracing context on current thread to underlying worker as parent tracing context. +/// +/// If you implement your own job/task scheduling upon global thread pool or schedules a long time running job in a infinite loop way, +/// you need to use class, or you need to use ThreadFromGlobalPool below. +/// +/// See the comments of ThreadPool below to know how it works. +using ThreadFromGlobalPoolNoTracingContextPropagation = ThreadFromGlobalPoolImpl; + +/// An alias of thread that execute jobs/tasks on global thread pool by implicit passing tracing context on current thread to underlying worker as parent tracing context. +/// If jobs/tasks are directly scheduled by using APIs of this class, you need to use this class or you need to use class above. +using ThreadFromGlobalPool = ThreadFromGlobalPoolImpl; + /// Recommended thread pool for the case when multiple thread pools are created and destroyed. /// /// The template parameter of ThreadFromGlobalPool is set to false to disable tracing context propagation to underlying worker. @@ -274,9 +286,6 @@ protected: /// which means the tracing context initialized at underlying worker level won't be delete for a very long time. /// This would cause wrong context for further jobs scheduled in ThreadPool. /// -/// To make sure the tracing context are correctly propagated, we explicitly disable context propagation(including initialization and de-initialization) at underlying worker level. +/// To make sure the tracing context is correctly propagated, we explicitly disable context propagation(including initialization and de-initialization) at underlying worker level. /// -using ThreadPool = ThreadPoolImpl>; - -/// An alias for user code to execute a job in the global thread pool -using ThreadFromGlobalPool = ThreadFromGlobalPoolImpl; +using ThreadPool = ThreadPoolImpl; diff --git a/src/Common/ZooKeeper/ZooKeeper.cpp b/src/Common/ZooKeeper/ZooKeeper.cpp index 6fcd3b52f16..55b793c2a70 100644 --- a/src/Common/ZooKeeper/ZooKeeper.cpp +++ b/src/Common/ZooKeeper/ZooKeeper.cpp @@ -605,7 +605,7 @@ void ZooKeeper::removeChildren(const std::string & path) } -void ZooKeeper::removeChildrenRecursive(const std::string & path, const String & keep_child_node) +void ZooKeeper::removeChildrenRecursive(const std::string & path, RemoveException keep_child) { Strings children = getChildren(path); while (!children.empty()) @@ -613,16 +613,23 @@ void ZooKeeper::removeChildrenRecursive(const std::string & path, const String & Coordination::Requests ops; for (size_t i = 0; i < MULTI_BATCH_SIZE && !children.empty(); ++i) { - removeChildrenRecursive(fs::path(path) / children.back()); - if (likely(keep_child_node.empty() || keep_child_node != children.back())) + if (keep_child.path.empty() || keep_child.path != children.back()) [[likely]] + { + removeChildrenRecursive(fs::path(path) / children.back()); ops.emplace_back(makeRemoveRequest(fs::path(path) / children.back(), -1)); + } + else if (keep_child.remove_subtree) + { + removeChildrenRecursive(fs::path(path) / children.back()); + } + children.pop_back(); } multi(ops); } } -bool ZooKeeper::tryRemoveChildrenRecursive(const std::string & path, bool probably_flat, const String & keep_child_node) +bool ZooKeeper::tryRemoveChildrenRecursive(const std::string & path, bool probably_flat, RemoveException keep_child) { Strings children; if (tryGetChildren(path, children) != Coordination::Error::ZOK) @@ -639,16 +646,20 @@ bool ZooKeeper::tryRemoveChildrenRecursive(const std::string & path, bool probab { String child_path = fs::path(path) / children.back(); - /// Will try to avoid recursive getChildren calls if child_path probably has no children. - /// It may be extremely slow when path contain a lot of leaf children. - if (!probably_flat) - tryRemoveChildrenRecursive(child_path); - - if (likely(keep_child_node.empty() || keep_child_node != children.back())) + if (keep_child.path.empty() || keep_child.path != children.back()) [[likely]] { + /// Will try to avoid recursive getChildren calls if child_path probably has no children. + /// It may be extremely slow when path contain a lot of leaf children. + if (!probably_flat) + tryRemoveChildrenRecursive(child_path); + batch.push_back(child_path); ops.emplace_back(zkutil::makeRemoveRequest(child_path, -1)); } + else if (keep_child.remove_subtree && !probably_flat) + { + tryRemoveChildrenRecursive(child_path); + } children.pop_back(); } diff --git a/src/Common/ZooKeeper/ZooKeeper.h b/src/Common/ZooKeeper/ZooKeeper.h index 1c7ba7f1d9c..791ae48b3f0 100644 --- a/src/Common/ZooKeeper/ZooKeeper.h +++ b/src/Common/ZooKeeper/ZooKeeper.h @@ -58,6 +58,18 @@ struct ShuffleHost } }; +struct RemoveException +{ + explicit RemoveException(std::string_view path_ = "", bool remove_subtree_ = true) + : path(path_) + , remove_subtree(remove_subtree_) + {} + + std::string_view path; + // whether we should keep the child node and its subtree or just the child node + bool remove_subtree; +}; + using GetPriorityForLoadBalancing = DB::GetPriorityForLoadBalancing; /// ZooKeeper session. The interface is substantially different from the usual libzookeeper API. @@ -219,13 +231,13 @@ public: void tryRemoveRecursive(const std::string & path); /// Similar to removeRecursive(...) and tryRemoveRecursive(...), but does not remove path itself. - /// If keep_child_node is not empty, this method will not remove path/keep_child_node (but will remove its subtree). - /// It can be useful to keep some child node as a flag which indicates that path is currently removing. - void removeChildrenRecursive(const std::string & path, const String & keep_child_node = {}); + /// Node defined as RemoveException will not be deleted. + void removeChildrenRecursive(const std::string & path, RemoveException keep_child = RemoveException{}); /// If probably_flat is true, this method will optimistically try to remove children non-recursive /// and will fall back to recursive removal if it gets ZNOTEMPTY for some child. /// Returns true if no kind of fallback happened. - bool tryRemoveChildrenRecursive(const std::string & path, bool probably_flat = false, const String & keep_child_node = {}); + /// Node defined as RemoveException will not be deleted. + bool tryRemoveChildrenRecursive(const std::string & path, bool probably_flat = false, RemoveException keep_child= RemoveException{}); /// Remove all children nodes (non recursive). void removeChildren(const std::string & path); diff --git a/src/Common/examples/compact_array.cpp b/src/Common/examples/compact_array.cpp index af6257e1963..58c4ea3be1e 100644 --- a/src/Common/examples/compact_array.cpp +++ b/src/Common/examples/compact_array.cpp @@ -1,9 +1,3 @@ -/// Bug in GCC: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=59124 -#if !defined(__clang__) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Warray-bounds" -#endif - #include #include #include @@ -262,7 +256,3 @@ int main() runTests(); return 0; } - -#if !defined(__clang__) -#pragma GCC diagnostic pop -#endif diff --git a/src/Common/examples/parallel_aggregation.cpp b/src/Common/examples/parallel_aggregation.cpp index 045a385671b..f54c4cee12c 100644 --- a/src/Common/examples/parallel_aggregation.cpp +++ b/src/Common/examples/parallel_aggregation.cpp @@ -69,11 +69,6 @@ static void aggregate1(Map & map, Source::const_iterator begin, Source::const_it ++map[*it]; } -#if !defined(__clang__) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" -#endif - static void aggregate12(Map & map, Source::const_iterator begin, Source::const_iterator end) { Map::LookupResult found = nullptr; @@ -122,10 +117,6 @@ static void aggregate22(MapTwoLevel & map, Source::const_iterator begin, Source: } } -#if !defined(__clang__) -#pragma GCC diagnostic pop -#endif - static void merge2(MapTwoLevel * maps, size_t num_threads, size_t bucket) { for (size_t i = 1; i < num_threads; ++i) diff --git a/src/Common/examples/parallel_aggregation2.cpp b/src/Common/examples/parallel_aggregation2.cpp index 496331e203d..6c20f46ab0e 100644 --- a/src/Common/examples/parallel_aggregation2.cpp +++ b/src/Common/examples/parallel_aggregation2.cpp @@ -62,11 +62,6 @@ struct AggregateIndependent } }; -#if !defined(__clang__) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" -#endif - template struct AggregateIndependentWithSequentialKeysOptimization { @@ -115,11 +110,6 @@ struct AggregateIndependentWithSequentialKeysOptimization } }; -#if !defined(__clang__) -#pragma GCC diagnostic pop -#endif - - template struct MergeSequential { @@ -265,20 +255,11 @@ struct Creator void operator()(Value &) const {} }; -#if !defined(__clang__) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" -#endif - struct Updater { void operator()(Value & x) const { ++x; } }; -#if !defined(__clang__) -#pragma GCC diagnostic pop -#endif - struct Merger { void operator()(Value & dst, const Value & src) const { dst += src; } diff --git a/src/Coordination/CoordinationSettings.h b/src/Coordination/CoordinationSettings.h index 5247f5d7ec8..c436c1b6635 100644 --- a/src/Coordination/CoordinationSettings.h +++ b/src/Coordination/CoordinationSettings.h @@ -30,6 +30,7 @@ struct Settings; M(UInt64, snapshot_distance, 100000, "How many log items we have to collect to write new snapshot", 0) \ M(Bool, auto_forwarding, true, "Allow to forward write requests from followers to leader", 0) \ M(Milliseconds, shutdown_timeout, 5000, "How much time we will wait until RAFT shutdown", 0) \ + M(Milliseconds, session_shutdown_timeout, 10000, "How much time we will wait until sessions are closed during shutdown", 0) \ M(Milliseconds, startup_timeout, 180000, "How much time we will wait until RAFT to start.", 0) \ M(LogsLevel, raft_logs_level, LogsLevel::information, "Log internal RAFT logs into main server log level. Valid values: 'trace', 'debug', 'information', 'warning', 'error', 'fatal', 'none'", 0) \ M(UInt64, rotate_log_storage_interval, 100000, "How many records will be stored in one log storage file", 0) \ diff --git a/src/Coordination/KeeperDispatcher.cpp b/src/Coordination/KeeperDispatcher.cpp index 5b376a03b02..261e43d80e4 100644 --- a/src/Coordination/KeeperDispatcher.cpp +++ b/src/Coordination/KeeperDispatcher.cpp @@ -354,9 +354,6 @@ void KeeperDispatcher::shutdown() update_configuration_thread.join(); } - if (server) - server->shutdown(); - KeeperStorage::RequestForSession request_for_session; /// Set session expired for all pending requests @@ -368,10 +365,58 @@ void KeeperDispatcher::shutdown() setResponse(request_for_session.session_id, response); } - /// Clear all registered sessions - std::lock_guard lock(session_to_response_callback_mutex); - session_to_response_callback.clear(); + KeeperStorage::RequestsForSessions close_requests; + { + /// Clear all registered sessions + std::lock_guard lock(session_to_response_callback_mutex); + + if (hasLeader()) + { + close_requests.reserve(session_to_response_callback.size()); + // send to leader CLOSE requests for active sessions + for (const auto & [session, response] : session_to_response_callback) + { + auto request = Coordination::ZooKeeperRequestFactory::instance().get(Coordination::OpNum::Close); + request->xid = Coordination::CLOSE_XID; + using namespace std::chrono; + KeeperStorage::RequestForSession request_info + { + .session_id = session, + .time = duration_cast(system_clock::now().time_since_epoch()).count(), + .request = std::move(request), + }; + + close_requests.push_back(std::move(request_info)); + } + } + + session_to_response_callback.clear(); + } + + // if there is no leader, there is no reason to do CLOSE because it's a write request + if (hasLeader() && !close_requests.empty()) + { + LOG_INFO(log, "Trying to close {} session(s)", close_requests.size()); + const auto raft_result = server->putRequestBatch(close_requests); + auto sessions_closing_done_promise = std::make_shared>(); + auto sessions_closing_done = sessions_closing_done_promise->get_future(); + raft_result->when_ready([sessions_closing_done_promise = std::move(sessions_closing_done_promise)]( + nuraft::cmd_result> & /*result*/, + nuraft::ptr & /*exception*/) { sessions_closing_done_promise->set_value(); }); + + auto session_shutdown_timeout = configuration_and_settings->coordination_settings->session_shutdown_timeout.totalMilliseconds(); + if (sessions_closing_done.wait_for(std::chrono::milliseconds(session_shutdown_timeout)) != std::future_status::ready) + LOG_WARNING( + log, + "Failed to close sessions in {}ms. If they are not closed, they will be closed after session timeout.", + session_shutdown_timeout); + } + + if (server) + server->shutdown(); + CurrentMetrics::set(CurrentMetrics::KeeperAliveConnections, 0); + } catch (...) { @@ -418,13 +463,15 @@ void KeeperDispatcher::sessionCleanerTask() LOG_INFO(log, "Found dead session {}, will try to close it", dead_session); /// Close session == send close request to raft server - Coordination::ZooKeeperRequestPtr request = Coordination::ZooKeeperRequestFactory::instance().get(Coordination::OpNum::Close); + auto request = Coordination::ZooKeeperRequestFactory::instance().get(Coordination::OpNum::Close); request->xid = Coordination::CLOSE_XID; - KeeperStorage::RequestForSession request_info; - request_info.request = request; using namespace std::chrono; - request_info.time = duration_cast(system_clock::now().time_since_epoch()).count(); - request_info.session_id = dead_session; + KeeperStorage::RequestForSession request_info + { + .session_id = dead_session, + .time = duration_cast(system_clock::now().time_since_epoch()).count(), + .request = std::move(request), + }; { std::lock_guard lock(push_request_mutex); if (!requests_queue->push(std::move(request_info))) diff --git a/src/Coordination/KeeperStateManager.cpp b/src/Coordination/KeeperStateManager.cpp index 3d7f5f2fb34..9b6aab5533e 100644 --- a/src/Coordination/KeeperStateManager.cpp +++ b/src/Coordination/KeeperStateManager.cpp @@ -6,6 +6,7 @@ #include #include #include +#include namespace DB { @@ -94,6 +95,14 @@ KeeperStateManager::parseServersConfiguration(const Poco::Util::AbstractConfigur continue; std::string full_prefix = config_prefix + ".raft_configuration." + server_key; + + if (getMultipleValuesFromConfig(config, full_prefix, "id").size() > 1 + || getMultipleValuesFromConfig(config, full_prefix, "hostname").size() > 1 + || getMultipleValuesFromConfig(config, full_prefix, "port").size() > 1) + { + throw Exception(ErrorCodes::RAFT_ERROR, "Multiple or or specified for a single "); + } + int new_server_id = config.getInt(full_prefix + ".id"); std::string hostname = config.getString(full_prefix + ".hostname"); int port = config.getInt(full_prefix + ".port"); diff --git a/src/Coordination/KeeperStorage.cpp b/src/Coordination/KeeperStorage.cpp index 9b5d7dc5db3..2328bc185a1 100644 --- a/src/Coordination/KeeperStorage.cpp +++ b/src/Coordination/KeeperStorage.cpp @@ -1,11 +1,11 @@ #include #include -#include #include #include #include -#include #include + +#include #include #include #include @@ -15,8 +15,11 @@ #include #include #include + #include #include +#include + #include #include #include @@ -36,17 +39,6 @@ namespace ErrorCodes namespace { -String base64Encode(const String & decoded) -{ - std::ostringstream ostr; // STYLE_CHECK_ALLOW_STD_STRING_STREAM - ostr.exceptions(std::ios::failbit); - Poco::Base64Encoder encoder(ostr); - encoder.rdbuf()->setLineLength(0); - encoder << decoded; - encoder.close(); - return ostr.str(); -} - String getSHA1(const String & userdata) { Poco::SHA1Engine engine; diff --git a/src/Core/BackgroundSchedulePool.cpp b/src/Core/BackgroundSchedulePool.cpp index b7a33c4930d..29cd3c1c540 100644 --- a/src/Core/BackgroundSchedulePool.cpp +++ b/src/Core/BackgroundSchedulePool.cpp @@ -149,9 +149,9 @@ BackgroundSchedulePool::BackgroundSchedulePool(size_t size_, CurrentMetrics::Met threads.resize(size_); for (auto & thread : threads) - thread = ThreadFromGlobalPool([this] { threadFunction(); }); + thread = ThreadFromGlobalPoolNoTracingContextPropagation([this] { threadFunction(); }); - delayed_thread = ThreadFromGlobalPool([this] { delayExecutionThreadFunction(); }); + delayed_thread = ThreadFromGlobalPoolNoTracingContextPropagation([this] { delayExecutionThreadFunction(); }); } @@ -168,7 +168,7 @@ void BackgroundSchedulePool::increaseThreadsCount(size_t new_threads_count) threads.resize(new_threads_count); for (size_t i = old_threads_count; i < new_threads_count; ++i) - threads[i] = ThreadFromGlobalPool([this] { threadFunction(); }); + threads[i] = ThreadFromGlobalPoolNoTracingContextPropagation([this] { threadFunction(); }); } diff --git a/src/Core/BackgroundSchedulePool.h b/src/Core/BackgroundSchedulePool.h index 36cbad145c9..1001d98e643 100644 --- a/src/Core/BackgroundSchedulePool.h +++ b/src/Core/BackgroundSchedulePool.h @@ -57,7 +57,9 @@ public: ~BackgroundSchedulePool(); private: - using Threads = std::vector; + /// BackgroundSchedulePool schedules a task on its own task queue, there's no need to construct/restore tracing context on this level. + /// This is also how ThreadPool class treats the tracing context. See ThreadPool for more information. + using Threads = std::vector; void threadFunction(); void delayExecutionThreadFunction(); @@ -83,7 +85,7 @@ private: std::condition_variable delayed_tasks_cond_var; std::mutex delayed_tasks_mutex; /// Thread waiting for next delayed task. - ThreadFromGlobalPool delayed_thread; + ThreadFromGlobalPoolNoTracingContextPropagation delayed_thread; /// Tasks ordered by scheduled time. DelayedTasks delayed_tasks; diff --git a/src/Core/Block.cpp b/src/Core/Block.cpp index 3b7595eb886..33691e83d27 100644 --- a/src/Core/Block.cpp +++ b/src/Core/Block.cpp @@ -623,6 +623,7 @@ NamesAndTypesList Block::getNamesAndTypesList() const NamesAndTypes Block::getNamesAndTypes() const { NamesAndTypes res; + res.reserve(columns()); for (const auto & elem : data) res.emplace_back(elem.name, elem.type); diff --git a/src/Core/Field.h b/src/Core/Field.h index a0945b8315a..2924ed9f174 100644 --- a/src/Core/Field.h +++ b/src/Core/Field.h @@ -105,10 +105,6 @@ template bool decimalEqual(T x, T y, UInt32 x_scale, UInt32 y_scale template bool decimalLess(T x, T y, UInt32 x_scale, UInt32 y_scale); template bool decimalLessOrEqual(T x, T y, UInt32 x_scale, UInt32 y_scale); -#if !defined(__clang__) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" -#endif template class DecimalField { @@ -168,9 +164,6 @@ private: T dec; UInt32 scale; }; -#if !defined(__clang__) -#pragma GCC diagnostic pop -#endif template constexpr bool is_decimal_field = false; template <> constexpr inline bool is_decimal_field> = true; @@ -432,16 +425,6 @@ public: bool isNegativeInfinity() const { return which == Types::Null && get().isNegativeInfinity(); } bool isPositiveInfinity() const { return which == Types::Null && get().isPositiveInfinity(); } - template - T & reinterpret(); - - template - const T & reinterpret() const - { - auto * mutable_this = const_cast *>(this); - return mutable_this->reinterpret(); - } - template bool tryGet(T & result) { const Types::Which requested = TypeToEnum>::value; @@ -559,7 +542,7 @@ public: case Types::Float64: { // Compare as UInt64 so that NaNs compare as equal. - return reinterpret() == rhs.reinterpret(); + return std::bit_cast(get()) == std::bit_cast(rhs.get()); } case Types::UUID: return get() == rhs.get(); case Types::String: return get() == rhs.get(); @@ -594,11 +577,6 @@ public: switch (field.which) { case Types::Null: return f(field.template get()); -// gcc 8.2.1 -#if !defined(__clang__) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" -#endif case Types::UInt64: return f(field.template get()); case Types::UInt128: return f(field.template get()); case Types::UInt256: return f(field.template get()); @@ -622,9 +600,6 @@ public: case Types::Decimal128: return f(field.template get>()); case Types::Decimal256: return f(field.template get>()); case Types::AggregateFunctionState: return f(field.template get()); -#if !defined(__clang__) -#pragma GCC diagnostic pop -#endif } __builtin_unreachable(); @@ -858,30 +833,6 @@ auto & Field::safeGet() } -template -T & Field::reinterpret() -{ - assert(which != Types::String); // See specialization for char - using ValueType = std::decay_t; - ValueType * MAY_ALIAS ptr = reinterpret_cast(&storage); - return *ptr; -} - -// Specialize reinterpreting to char (used in ColumnUnique) to make sure Strings are reinterpreted correctly -// inline to avoid multiple definitions -template <> -inline char & Field::reinterpret() -{ - if (which == Types::String) - { - // For String we want to return a pointer to the data, not the start of the class - // as the layout of std::string depends on the STD version and options - char * ptr = reinterpret_cast(&storage)->data(); - return *ptr; - } - return *reinterpret_cast(&storage); -} - template Field::Field(T && rhs, enable_if_not_field_or_bool_or_stringlike_t) //-V730 { diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 47c86295a34..870647b3254 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -136,6 +136,7 @@ static constexpr UInt64 operator""_GiB(unsigned long long value) M(Bool, distributed_aggregation_memory_efficient, true, "Is the memory-saving mode of distributed aggregation enabled.", 0) \ M(UInt64, aggregation_memory_efficient_merge_threads, 0, "Number of threads to use for merge intermediate aggregation results in memory efficient mode. When bigger, then more memory is consumed. 0 means - same as 'max_threads'.", 0) \ M(Bool, enable_positional_arguments, true, "Enable positional arguments in ORDER BY, GROUP BY and LIMIT BY", 0) \ + M(Bool, enable_extended_results_for_datetime_functions, false, "Enable date functions like toLastDayOfMonth return Date32 results (instead of Date results) for Date32/DateTime64 arguments.", 0) \ \ M(Bool, group_by_use_nulls, false, "Treat columns mentioned in ROLLUP, CUBE or GROUPING SETS as Nullable", 0) \ \ @@ -481,7 +482,7 @@ static constexpr UInt64 operator""_GiB(unsigned long long value) M(Bool, optimize_if_chain_to_multiif, false, "Replace if(cond1, then1, if(cond2, ...)) chains to multiIf. Currently it's not beneficial for numeric types.", 0) \ M(Bool, optimize_multiif_to_if, true, "Replace 'multiIf' with only one condition to 'if'.", 0) \ M(Bool, optimize_if_transform_strings_to_enum, false, "Replaces string-type arguments in If and Transform to enum. Disabled by default cause it could make inconsistent change in distributed query that would lead to its fail.", 0) \ - M(Bool, optimize_monotonous_functions_in_order_by, true, "Replace monotonous function with its argument in ORDER BY", 0) \ + M(Bool, optimize_monotonous_functions_in_order_by, false, "Replace monotonous function with its argument in ORDER BY", 0) \ M(Bool, optimize_functions_to_subcolumns, false, "Transform functions to subcolumns, if possible, to reduce amount of read data. E.g. 'length(arr)' -> 'arr.size0', 'col IS NULL' -> 'col.null' ", 0) \ M(Bool, optimize_using_constraints, false, "Use constraints for query optimization", 0) \ M(Bool, optimize_substitute_columns, false, "Use constraints for column substitution", 0) \ @@ -527,7 +528,7 @@ static constexpr UInt64 operator""_GiB(unsigned long long value) M(Bool, describe_extend_object_types, false, "Deduce concrete type of columns of type Object in DESCRIBE query", 0) \ M(Bool, describe_include_subcolumns, false, "If true, subcolumns of all table columns will be included into result of DESCRIBE query", 0) \ \ - M(Bool, optimize_rewrite_sum_if_to_count_if, true, "Rewrite sumIf() and sum(if()) function countIf() function when logically equivalent", 0) \ + M(Bool, optimize_rewrite_sum_if_to_count_if, false, "Rewrite sumIf() and sum(if()) function countIf() function when logically equivalent", 0) \ M(UInt64, insert_shard_id, 0, "If non zero, when insert into a distributed table, the data will be inserted into the shard `insert_shard_id` synchronously. Possible values range from 1 to `shards_number` of corresponding distributed table", 0) \ \ M(Bool, collect_hash_table_stats_during_aggregation, true, "Enable collecting hash table statistics to optimize memory allocation", 0) \ @@ -618,6 +619,8 @@ static constexpr UInt64 operator""_GiB(unsigned long long value) M(Bool, allow_deprecated_database_ordinary, false, "Allow to create databases with deprecated Ordinary engine", 0) \ M(Bool, allow_deprecated_syntax_for_merge_tree, false, "Allow to create *MergeTree tables with deprecated engine definition syntax", 0) \ \ + M(Bool, force_grouping_standard_compatibility, true, "Make GROUPING function to return 1 when argument is not used as an aggregation key", 0) \ + \ M(Bool, schema_inference_use_cache_for_file, true, "Use cache in schema inference while using file table function", 0) \ M(Bool, schema_inference_use_cache_for_s3, true, "Use cache in schema inference while using s3 table function", 0) \ M(Bool, schema_inference_use_cache_for_hdfs, true, "Use cache in schema inference while using hdfs table function", 0) \ @@ -775,6 +778,8 @@ static constexpr UInt64 operator""_GiB(unsigned long long value) \ M(UInt64, input_format_allow_errors_num, 0, "Maximum absolute amount of errors while reading text formats (like CSV, TSV). In case of error, if at least absolute or relative amount of errors is lower than corresponding value, will skip until next line and continue.", 0) \ M(Float, input_format_allow_errors_ratio, 0, "Maximum relative amount of errors while reading text formats (like CSV, TSV). In case of error, if at least absolute or relative amount of errors is lower than corresponding value, will skip until next line and continue.", 0) \ + M(String, input_format_record_errors_file_path, "", "Path of the file used to record errors while reading text formats (CSV, TSV).", 0) \ + M(String, errors_output_format, "CSV", "Method to write Errors to text output.", 0) \ \ M(String, format_schema, "", "Schema identifier (used by schema-based formats)", 0) \ M(String, format_template_resultset, "", "Path to file which contains format string for result set (for Template format)", 0) \ diff --git a/src/Core/SettingsChangesHistory.h b/src/Core/SettingsChangesHistory.h index be2def2c01a..b78b812da86 100644 --- a/src/Core/SettingsChangesHistory.h +++ b/src/Core/SettingsChangesHistory.h @@ -78,6 +78,7 @@ namespace SettingsChangesHistory /// It's used to implement `compatibility` setting (see https://github.com/ClickHouse/ClickHouse/issues/35972) static std::map settings_changes_history = { + {"22.9", {{"force_grouping_standard_compatibility", false, true, "Make GROUPING function output the same as in SQL standard and other DBMS"}}}, {"22.7", {{"cross_to_inner_join_rewrite", 1, 2, "Force rewrite comma join to inner"}, {"enable_positional_arguments", false, true, "Enable positional arguments feature by default"}, {"format_csv_allow_single_quotes", true, false, "Most tools don't treat single quote in CSV specially, don't do it by default too"}}}, diff --git a/src/Core/Types.h b/src/Core/Types.h index 92546d7d07a..0dfc089f144 100644 --- a/src/Core/Types.h +++ b/src/Core/Types.h @@ -42,11 +42,6 @@ struct Null } }; -/// Ignore strange gcc warning https://gcc.gnu.org/bugzilla/show_bug.cgi?id=55776 -#if !defined(__clang__) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wshadow" -#endif /// @note Except explicitly described you should not assume on TypeIndex numbers and/or their orders in this enum. enum class TypeIndex { @@ -89,9 +84,6 @@ enum class TypeIndex Map, Object, }; -#if !defined(__clang__) -#pragma GCC diagnostic pop -#endif using UInt128 = ::UInt128; diff --git a/src/Core/examples/coro.cpp b/src/Core/examples/coro.cpp index 370820a228d..fbccc261e9d 100644 --- a/src/Core/examples/coro.cpp +++ b/src/Core/examples/coro.cpp @@ -14,7 +14,7 @@ namespace std // NOLINT(cert-dcl58-cpp) { - using namespace experimental::coroutines_v1; + using namespace experimental::coroutines_v1; // NOLINT(cert-dcl58-cpp) } #if __has_warning("-Wdeprecated-experimental-coroutine") diff --git a/src/DataTypes/ObjectUtils.cpp b/src/DataTypes/ObjectUtils.cpp index 3cf557ec5bf..e5d8d05acb5 100644 --- a/src/DataTypes/ObjectUtils.cpp +++ b/src/DataTypes/ObjectUtils.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -159,6 +160,16 @@ void convertObjectsToTuples(Block & block, const NamesAndTypesList & extended_st } } +void deduceTypesOfObjectColumns(const StorageSnapshotPtr & storage_snapshot, Block & block) +{ + if (!storage_snapshot->object_columns.empty()) + { + auto options = GetColumnsOptions(GetColumnsOptions::AllPhysical).withExtendedObjects(); + auto storage_columns = storage_snapshot->getColumns(options); + convertObjectsToTuples(block, storage_columns); + } +} + static bool isPrefix(const PathInData::Parts & prefix, const PathInData::Parts & parts) { if (prefix.size() > parts.size()) @@ -442,15 +453,19 @@ using SubcolumnsTreeWithColumns = SubcolumnsTree; using Node = SubcolumnsTreeWithColumns::Node; /// Creates data type and column from tree of subcolumns. -ColumnWithTypeAndDimensions createTypeFromNode(const Node * node) +ColumnWithTypeAndDimensions createTypeFromNode(const Node & node) { auto collect_tuple_elemets = [](const auto & children) { + if (children.empty()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot create type from empty Tuple or Nested node"); + std::vector> tuple_elements; tuple_elements.reserve(children.size()); for (const auto & [name, child] : children) { - auto column = createTypeFromNode(child.get()); + assert(child); + auto column = createTypeFromNode(*child); tuple_elements.emplace_back(name, std::move(column)); } @@ -464,13 +479,13 @@ ColumnWithTypeAndDimensions createTypeFromNode(const Node * node) return std::make_tuple(std::move(tuple_names), std::move(tuple_columns)); }; - if (node->kind == Node::SCALAR) + if (node.kind == Node::SCALAR) { - return node->data; + return node.data; } - else if (node->kind == Node::NESTED) + else if (node.kind == Node::NESTED) { - auto [tuple_names, tuple_columns] = collect_tuple_elemets(node->children); + auto [tuple_names, tuple_columns] = collect_tuple_elemets(node.children); Columns offsets_columns; offsets_columns.reserve(tuple_columns[0].array_dimensions + 1); @@ -481,7 +496,7 @@ ColumnWithTypeAndDimensions createTypeFromNode(const Node * node) /// `k1 Array(Nested(k2 Int, k3 Int))` and k1 is marked as Nested /// and `k2` and `k3` has anonymous_array_level = 1 in that case. - const auto & current_array = assert_cast(*node->data.column); + const auto & current_array = assert_cast(*node.data.column); offsets_columns.push_back(current_array.getOffsetsPtr()); auto first_column = tuple_columns[0].column; @@ -518,7 +533,7 @@ ColumnWithTypeAndDimensions createTypeFromNode(const Node * node) } else { - auto [tuple_names, tuple_columns] = collect_tuple_elemets(node->children); + auto [tuple_names, tuple_columns] = collect_tuple_elemets(node.children); size_t num_elements = tuple_columns.size(); Columns tuple_elements_columns(num_elements); @@ -576,6 +591,15 @@ std::pair unflattenObjectToTuple(const ColumnObject & co { const auto & subcolumns = column.getSubcolumns(); + if (subcolumns.empty()) + { + auto type = std::make_shared( + DataTypes{std::make_shared()}, + Names{ColumnObject::COLUMN_NAME_DUMMY}); + + return {type->createColumn()->cloneResized(column.size()), type}; + } + PathsInData paths; DataTypes types; Columns columns; @@ -602,6 +626,9 @@ std::pair unflattenTuple( assert(paths.size() == tuple_types.size()); assert(paths.size() == tuple_columns.size()); + if (paths.empty()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot unflatten empty Tuple"); + /// We add all paths to the subcolumn tree and then create a type from it. /// The tree stores column, type and number of array dimensions /// for each intermediate node. diff --git a/src/DataTypes/ObjectUtils.h b/src/DataTypes/ObjectUtils.h index 2dde0ed3e65..c60d5bec208 100644 --- a/src/DataTypes/ObjectUtils.h +++ b/src/DataTypes/ObjectUtils.h @@ -11,6 +11,9 @@ namespace DB { +struct StorageSnapshot; +using StorageSnapshotPtr = std::shared_ptr; + /// Returns number of dimensions in Array type. 0 if type is not array. size_t getNumberOfDimensions(const IDataType & type); @@ -38,6 +41,7 @@ DataTypePtr getDataTypeByColumn(const IColumn & column); /// Converts Object types and columns to Tuples in @columns_list and @block /// and checks that types are consistent with types in @extended_storage_columns. void convertObjectsToTuples(Block & block, const NamesAndTypesList & extended_storage_columns); +void deduceTypesOfObjectColumns(const StorageSnapshotPtr & storage_snapshot, Block & block); /// Checks that each path is not the prefix of any other path. void checkObjectHasNoAmbiguosPaths(const PathsInData & paths); @@ -164,27 +168,24 @@ ColumnsDescription getObjectColumns( const ColumnsDescription & storage_columns, EntryColumnsGetter && entry_columns_getter) { - ColumnsDescription res; - - if (begin == end) - { - for (const auto & column : storage_columns) - { - if (isObject(column.type)) - { - auto tuple_type = std::make_shared( - DataTypes{std::make_shared()}, - Names{ColumnObject::COLUMN_NAME_DUMMY}); - - res.add({column.name, std::move(tuple_type)}); - } - } - - return res; - } - std::unordered_map types_in_entries; + /// Add dummy column for all Object columns + /// to not lose any column if it's missing + /// in all entries. If it exists in any entry + /// dummy column will be removed. + for (const auto & column : storage_columns) + { + if (isObject(column.type)) + { + auto tuple_type = std::make_shared( + DataTypes{std::make_shared()}, + Names{ColumnObject::COLUMN_NAME_DUMMY}); + + types_in_entries[column.name].push_back(std::move(tuple_type)); + } + } + for (auto it = begin; it != end; ++it) { const auto & entry_columns = entry_columns_getter(*it); @@ -196,6 +197,7 @@ ColumnsDescription getObjectColumns( } } + ColumnsDescription res; for (const auto & [name, types] : types_in_entries) res.add({name, getLeastCommonTypeForObject(types)}); diff --git a/src/DataTypes/Serializations/SerializationArray.cpp b/src/DataTypes/Serializations/SerializationArray.cpp index abd99038e98..eb93b5049a0 100644 --- a/src/DataTypes/Serializations/SerializationArray.cpp +++ b/src/DataTypes/Serializations/SerializationArray.cpp @@ -20,8 +20,13 @@ namespace ErrorCodes extern const int CANNOT_READ_ALL_DATA; extern const int CANNOT_READ_ARRAY_FROM_TEXT; extern const int LOGICAL_ERROR; + extern const int TOO_LARGE_ARRAY_SIZE; } +static constexpr size_t MAX_ARRAY_SIZE = 1ULL << 30; +static constexpr size_t MAX_ARRAYS_SIZE = 1ULL << 40; + + void SerializationArray::serializeBinary(const Field & field, WriteBuffer & ostr) const { const Array & a = field.get(); @@ -125,7 +130,12 @@ namespace { ColumnArray::Offset current_size = 0; readIntBinary(current_size, istr); - current_offset += current_size; + + if (unlikely(current_size > MAX_ARRAY_SIZE)) + throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE, "Array size is too large: {}", current_size); + if (unlikely(__builtin_add_overflow(current_offset, current_size, ¤t_offset))) + throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE, "Deserialization of array offsets will lead to overflow"); + offset_values[i] = current_offset; ++i; } @@ -174,7 +184,7 @@ namespace { auto current_offset = offsets_data[i]; sizes_data[i] = current_offset - prev_offset; - prev_offset = current_offset; + prev_offset = current_offset; } return column_sizes; @@ -348,6 +358,9 @@ void SerializationArray::deserializeBinaryBulkWithMultipleStreams( throw Exception("Nested column is longer than last offset", ErrorCodes::LOGICAL_ERROR); size_t nested_limit = last_offset - nested_column->size(); + if (unlikely(nested_limit > MAX_ARRAYS_SIZE)) + throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE, "Array sizes are too large: {}", nested_limit); + /// Adjust value size hint. Divide it to the average array size. settings.avg_value_size_hint = nested_limit ? settings.avg_value_size_hint / nested_limit * offset_values.size() : 0; diff --git a/src/DataTypes/Serializations/SerializationFixedString.cpp b/src/DataTypes/Serializations/SerializationFixedString.cpp index 9baaf95cb52..dd476103108 100644 --- a/src/DataTypes/Serializations/SerializationFixedString.cpp +++ b/src/DataTypes/Serializations/SerializationFixedString.cpp @@ -24,6 +24,8 @@ namespace ErrorCodes extern const int TOO_LARGE_STRING_SIZE; } +static constexpr size_t MAX_STRINGS_SIZE = 1ULL << 30; + void SerializationFixedString::serializeBinary(const Field & field, WriteBuffer & ostr) const { const String & s = field.get(); @@ -85,8 +87,17 @@ void SerializationFixedString::deserializeBinaryBulk(IColumn & column, ReadBuffe ColumnFixedString::Chars & data = typeid_cast(column).getChars(); size_t initial_size = data.size(); - size_t max_bytes = limit * n; - data.resize(initial_size + max_bytes); + size_t max_bytes; + size_t new_data_size; + + if (unlikely(__builtin_mul_overflow(limit, n, &max_bytes))) + throw Exception(ErrorCodes::TOO_LARGE_STRING_SIZE, "Deserializing FixedString will lead to overflow"); + if (unlikely(max_bytes > MAX_STRINGS_SIZE)) + throw Exception(ErrorCodes::TOO_LARGE_STRING_SIZE, "Too large sizes of FixedString to deserialize: {}", max_bytes); + if (unlikely(__builtin_add_overflow(initial_size, max_bytes, &new_data_size))) + throw Exception(ErrorCodes::TOO_LARGE_STRING_SIZE, "Deserializing FixedString will lead to overflow"); + + data.resize(new_data_size); size_t read_bytes = istr.readBig(reinterpret_cast(&data[initial_size]), max_bytes); if (read_bytes % n != 0) diff --git a/src/DataTypes/Serializations/SerializationString.cpp b/src/DataTypes/Serializations/SerializationString.cpp index b6c4523eb52..042f2b3d45b 100644 --- a/src/DataTypes/Serializations/SerializationString.cpp +++ b/src/DataTypes/Serializations/SerializationString.cpp @@ -17,6 +17,7 @@ #include #endif + namespace DB { diff --git a/src/DataTypes/Serializations/SubcolumnsTree.h b/src/DataTypes/Serializations/SubcolumnsTree.h index f66f557bc8f..fda45e1e9a2 100644 --- a/src/DataTypes/Serializations/SubcolumnsTree.h +++ b/src/DataTypes/Serializations/SubcolumnsTree.h @@ -51,6 +51,8 @@ public: using NodeKind = typename Node::Kind; using NodePtr = std::shared_ptr; + SubcolumnsTree() : root(std::make_shared(Node::TUPLE)) {} + /// Add a leaf without any data in other nodes. bool add(const PathInData & path, const NodeData & leaf_data) { @@ -73,13 +75,9 @@ public: bool add(const PathInData & path, const NodeCreator & node_creator) { const auto & parts = path.getParts(); - if (parts.empty()) return false; - if (!root) - root = std::make_shared(Node::TUPLE); - Node * current_node = root.get(); for (size_t i = 0; i < parts.size() - 1; ++i) { @@ -166,13 +164,13 @@ public: return node; } - bool empty() const { return root == nullptr; } + bool empty() const { return root->children.empty(); } size_t size() const { return leaves.size(); } using Nodes = std::vector; const Nodes & getLeaves() const { return leaves; } - const Node * getRoot() const { return root.get(); } + const Node & getRoot() const { return *root; } using iterator = typename Nodes::iterator; using const_iterator = typename Nodes::const_iterator; @@ -186,11 +184,11 @@ public: private: const Node * findImpl(const PathInData & path, bool find_exact) const { - if (!root) + if (empty()) return nullptr; const auto & parts = path.getParts(); - const Node * current_node = root.get(); + const auto * current_node = root.get(); for (const auto & part : parts) { diff --git a/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp b/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp index fa4a79415ec..da7f8c871cb 100644 --- a/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp +++ b/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp @@ -143,9 +143,11 @@ void CachedOnDiskReadBufferFromFile::initialize(size_t offset, size_t size) } CachedOnDiskReadBufferFromFile::ImplementationBufferPtr -CachedOnDiskReadBufferFromFile::getCacheReadBuffer(size_t offset) const +CachedOnDiskReadBufferFromFile::getCacheReadBuffer(const FileSegment & file_segment) const { - auto path = cache->getPathInLocalCache(cache_key, offset, is_persistent); + /// Use is_persistent flag from in-memory state of the filesegment, + /// because it is consistent with what is written on disk. + auto path = file_segment.getPathInLocalCache(); ReadSettings local_read_settings{settings}; /// Do not allow to use asynchronous version of LocalFSReadMethod. @@ -206,7 +208,7 @@ CachedOnDiskReadBufferFromFile::getRemoteFSReadBuffer(FileSegment & file_segment return remote_file_reader; auto remote_fs_segment_reader = file_segment.extractRemoteFileReader(); - if (remote_fs_segment_reader && file_offset_of_buffer_end == remote_file_reader->getFileOffsetOfBufferEnd()) + if (remote_fs_segment_reader && file_offset_of_buffer_end == remote_fs_segment_reader->getFileOffsetOfBufferEnd()) remote_file_reader = remote_fs_segment_reader; else remote_file_reader = implementation_buffer_creator(); @@ -237,8 +239,6 @@ bool CachedOnDiskReadBufferFromFile::canStartFromCache(size_t current_offset, co CachedOnDiskReadBufferFromFile::ImplementationBufferPtr CachedOnDiskReadBufferFromFile::getReadBufferForFileSegment(FileSegmentPtr & file_segment) { - auto range = file_segment->range(); - auto download_state = file_segment->state(); LOG_TEST(log, "getReadBufferForFileSegment: {}", file_segment->getInfoForLog()); @@ -247,7 +247,7 @@ CachedOnDiskReadBufferFromFile::getReadBufferForFileSegment(FileSegmentPtr & fil if (download_state == FileSegment::State::DOWNLOADED) { read_type = ReadType::CACHED; - return getCacheReadBuffer(range.left); + return getCacheReadBuffer(*file_segment); } else { @@ -280,7 +280,7 @@ CachedOnDiskReadBufferFromFile::getReadBufferForFileSegment(FileSegmentPtr & fil /// file_offset_of_buffer_end read_type = ReadType::CACHED; - return getCacheReadBuffer(range.left); + return getCacheReadBuffer(*file_segment); } download_state = file_segment->wait(); @@ -289,7 +289,7 @@ CachedOnDiskReadBufferFromFile::getReadBufferForFileSegment(FileSegmentPtr & fil case FileSegment::State::DOWNLOADED: { read_type = ReadType::CACHED; - return getCacheReadBuffer(range.left); + return getCacheReadBuffer(*file_segment); } case FileSegment::State::EMPTY: case FileSegment::State::PARTIALLY_DOWNLOADED: @@ -305,7 +305,7 @@ CachedOnDiskReadBufferFromFile::getReadBufferForFileSegment(FileSegmentPtr & fil /// file_offset_of_buffer_end read_type = ReadType::CACHED; - return getCacheReadBuffer(range.left); + return getCacheReadBuffer(*file_segment); } auto downloader_id = file_segment->getOrSetDownloader(); @@ -323,7 +323,7 @@ CachedOnDiskReadBufferFromFile::getReadBufferForFileSegment(FileSegmentPtr & fil read_type = ReadType::CACHED; file_segment->resetDownloader(); - return getCacheReadBuffer(range.left); + return getCacheReadBuffer(*file_segment); } if (file_segment->getCurrentWriteOffset() < file_offset_of_buffer_end) @@ -339,7 +339,7 @@ CachedOnDiskReadBufferFromFile::getReadBufferForFileSegment(FileSegmentPtr & fil LOG_TEST(log, "Predownload. File segment info: {}", file_segment->getInfoForLog()); chassert(file_offset_of_buffer_end > file_segment->getCurrentWriteOffset()); bytes_to_predownload = file_offset_of_buffer_end - file_segment->getCurrentWriteOffset(); - chassert(bytes_to_predownload < range.size()); + chassert(bytes_to_predownload < file_segment->range().size()); } read_type = ReadType::REMOTE_FS_READ_AND_PUT_IN_CACHE; @@ -354,7 +354,7 @@ CachedOnDiskReadBufferFromFile::getReadBufferForFileSegment(FileSegmentPtr & fil if (canStartFromCache(file_offset_of_buffer_end, *file_segment)) { read_type = ReadType::CACHED; - return getCacheReadBuffer(range.left); + return getCacheReadBuffer(*file_segment); } else { diff --git a/src/Disks/IO/CachedOnDiskReadBufferFromFile.h b/src/Disks/IO/CachedOnDiskReadBufferFromFile.h index b86e53ec160..535d01f3a8c 100644 --- a/src/Disks/IO/CachedOnDiskReadBufferFromFile.h +++ b/src/Disks/IO/CachedOnDiskReadBufferFromFile.h @@ -68,7 +68,7 @@ private: ImplementationBufferPtr getReadBufferForFileSegment(FileSegmentPtr & file_segment); - ImplementationBufferPtr getCacheReadBuffer(size_t offset) const; + ImplementationBufferPtr getCacheReadBuffer(const FileSegment & file_segment) const; std::optional getLastNonDownloadedOffset() const; diff --git a/src/Disks/IO/ThreadPoolReader.cpp b/src/Disks/IO/ThreadPoolReader.cpp index 9b38607c204..d2b3bcbaa5e 100644 --- a/src/Disks/IO/ThreadPoolReader.cpp +++ b/src/Disks/IO/ThreadPoolReader.cpp @@ -108,8 +108,19 @@ std::future ThreadPoolReader::submit(Request reques if (has_pread_nowait_support.load(std::memory_order_relaxed)) { + /// It reports real time spent including the time spent while thread was preempted doing nothing. + /// And it is Ok for the purpose of this watch (it is used to lower the number of threads to read from tables). + /// Sometimes it is better to use taskstats::blkio_delay_total, but it is quite expensive to get it + /// (TaskStatsInfoGetter has about 500K RPS). Stopwatch watch(CLOCK_MONOTONIC); + SCOPE_EXIT({ + watch.stop(); + + ProfileEvents::increment(ProfileEvents::ThreadPoolReaderPageCacheHitElapsedMicroseconds, watch.elapsedMicroseconds()); + ProfileEvents::increment(ProfileEvents::DiskReadElapsedMicroseconds, watch.elapsedMicroseconds()); + }); + std::promise promise; std::future future = promise.get_future(); @@ -135,11 +146,6 @@ std::future ThreadPoolReader::submit(Request reques { /// The file has ended. promise.set_value({0, 0}); - - watch.stop(); - ProfileEvents::increment(ProfileEvents::ThreadPoolReaderPageCacheHitElapsedMicroseconds, watch.elapsedMicroseconds()); - ProfileEvents::increment(ProfileEvents::DiskReadElapsedMicroseconds, watch.elapsedMicroseconds()); - return future; } @@ -179,18 +185,10 @@ std::future ThreadPoolReader::submit(Request reques if (bytes_read) { - /// It reports real time spent including the time spent while thread was preempted doing nothing. - /// And it is Ok for the purpose of this watch (it is used to lower the number of threads to read from tables). - /// Sometimes it is better to use taskstats::blkio_delay_total, but it is quite expensive to get it - /// (TaskStatsInfoGetter has about 500K RPS). - watch.stop(); - /// Read successfully from page cache. ProfileEvents::increment(ProfileEvents::ThreadPoolReaderPageCacheHit); ProfileEvents::increment(ProfileEvents::ThreadPoolReaderPageCacheHitBytes, bytes_read); ProfileEvents::increment(ProfileEvents::ReadBufferFromFileDescriptorReadBytes, bytes_read); - ProfileEvents::increment(ProfileEvents::ThreadPoolReaderPageCacheHitElapsedMicroseconds, watch.elapsedMicroseconds()); - ProfileEvents::increment(ProfileEvents::DiskReadElapsedMicroseconds, watch.elapsedMicroseconds()); promise.set_value({bytes_read, request.ignore}); return future; @@ -226,6 +224,12 @@ std::future ThreadPoolReader::submit(Request reques setThreadName("ThreadPoolRead"); Stopwatch watch(CLOCK_MONOTONIC); + SCOPE_EXIT({ + watch.stop(); + + ProfileEvents::increment(ProfileEvents::ThreadPoolReaderPageCacheMissElapsedMicroseconds, watch.elapsedMicroseconds()); + ProfileEvents::increment(ProfileEvents::DiskReadElapsedMicroseconds, watch.elapsedMicroseconds()); + }); size_t bytes_read = 0; while (!bytes_read) @@ -254,8 +258,6 @@ std::future ThreadPoolReader::submit(Request reques ProfileEvents::increment(ProfileEvents::ThreadPoolReaderPageCacheMissBytes, bytes_read); ProfileEvents::increment(ProfileEvents::ReadBufferFromFileDescriptorReadBytes, bytes_read); - ProfileEvents::increment(ProfileEvents::ThreadPoolReaderPageCacheMissElapsedMicroseconds, watch.elapsedMicroseconds()); - ProfileEvents::increment(ProfileEvents::DiskReadElapsedMicroseconds, watch.elapsedMicroseconds()); return Result{ .size = bytes_read, .offset = request.ignore }; }); diff --git a/src/Disks/ObjectStorages/DiskObjectStorage.cpp b/src/Disks/ObjectStorages/DiskObjectStorage.cpp index db8f90e777d..f4462a0f8e6 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorage.cpp +++ b/src/Disks/ObjectStorages/DiskObjectStorage.cpp @@ -156,6 +156,8 @@ void DiskObjectStorage::getRemotePathsRecursive(const String & local_path, std:: e.code() == ErrorCodes::ATTEMPT_TO_READ_AFTER_EOF || e.code() == ErrorCodes::CANNOT_READ_ALL_DATA) return; + + throw; } catch (const fs::filesystem_error & e) { diff --git a/src/Disks/ObjectStorages/DiskObjectStorageMetadata.cpp b/src/Disks/ObjectStorages/DiskObjectStorageMetadata.cpp index f18debe8a8b..56cc20098ba 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorageMetadata.cpp +++ b/src/Disks/ObjectStorages/DiskObjectStorageMetadata.cpp @@ -13,7 +13,6 @@ namespace DB namespace ErrorCodes { extern const int UNKNOWN_FORMAT; - extern const int LOGICAL_ERROR; } void DiskObjectStorageMetadata::deserialize(ReadBuffer & buf) @@ -131,9 +130,6 @@ DiskObjectStorageMetadata::DiskObjectStorageMetadata( void DiskObjectStorageMetadata::addObject(const String & path, size_t size) { - if (!object_storage_root_path.empty() && path.starts_with(object_storage_root_path)) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected relative path"); - total_size += size; storage_objects.emplace_back(path, size); } diff --git a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp index 45304ac2fac..998b521cc56 100644 --- a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp +++ b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp @@ -31,7 +31,6 @@ #include #include - namespace DB { @@ -91,19 +90,7 @@ void logIfError(const Aws::Utils::Outcome & response, std::functi std::string S3ObjectStorage::generateBlobNameForPath(const std::string & /* path */) { - /// Path to store the new S3 object. - - /// Total length is 32 a-z characters for enough randomness. - /// First 3 characters are used as a prefix for - /// https://aws.amazon.com/premiumsupport/knowledge-center/s3-object-key-naming-pattern/ - - constexpr size_t key_name_total_size = 32; - constexpr size_t key_name_prefix_size = 3; - - /// Path to store new S3 object. - return fmt::format("{}/{}", - getRandomASCIIString(key_name_prefix_size), - getRandomASCIIString(key_name_total_size - key_name_prefix_size)); + return getRandomASCIIString(32); } Aws::S3::Model::HeadObjectOutcome S3ObjectStorage::requestObjectHeadData(const std::string & bucket_from, const std::string & key) const diff --git a/src/Disks/ObjectStorages/StoredObject.h b/src/Disks/ObjectStorages/StoredObject.h index d9faa766540..acb8a5fd127 100644 --- a/src/Disks/ObjectStorages/StoredObject.h +++ b/src/Disks/ObjectStorages/StoredObject.h @@ -3,7 +3,6 @@ #include #include - namespace DB { diff --git a/src/Disks/tests/gtest_disk.cpp b/src/Disks/tests/gtest_disk.cpp index 36f91249391..908e76b5c63 100644 --- a/src/Disks/tests/gtest_disk.cpp +++ b/src/Disks/tests/gtest_disk.cpp @@ -7,12 +7,6 @@ namespace fs = std::filesystem; -#if !defined(__clang__) -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wsuggest-override" -#endif - - template DB::DiskPtr createDisk(); diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp index 780b6bb6201..3de4a0de391 100644 --- a/src/Formats/FormatFactory.cpp +++ b/src/Formats/FormatFactory.cpp @@ -243,11 +243,20 @@ InputFormatPtr FormatFactory::getInput( ParallelParsingInputFormat::Params params{ buf, sample, parser_creator, file_segmentation_engine, name, settings.max_threads, settings.min_chunk_bytes_for_parallel_parsing, context->getApplicationType() == Context::ApplicationType::SERVER}; - return std::make_shared(params); + auto format = std::make_shared(params); + if (!settings.input_format_record_errors_file_path.toString().empty()) + { + format->setErrorsLogger(std::make_shared(context)); + } + return format; } auto format = getInputFormat(name, buf, sample, context, max_block_size, format_settings); + if (!settings.input_format_record_errors_file_path.toString().empty()) + { + format->setErrorsLogger(std::make_shared(context)); + } return format; } @@ -521,6 +530,7 @@ String FormatFactory::getFormatFromFileDescriptor(int fd) return getFormatFromFileName(file_path, false); return ""; #else + (void)fd; return ""; #endif } diff --git a/src/Formats/ReadSchemaUtils.cpp b/src/Formats/ReadSchemaUtils.cpp index d09cb9ff9ad..8468f540253 100644 --- a/src/Formats/ReadSchemaUtils.cpp +++ b/src/Formats/ReadSchemaUtils.cpp @@ -63,9 +63,10 @@ ColumnsDescription readSchemaFromFormat( { names_and_types = external_schema_reader->readSchema(); } - catch (const DB::Exception & e) + catch (Exception & e) { - throw Exception(ErrorCodes::CANNOT_EXTRACT_TABLE_STRUCTURE, "Cannot extract table structure from {} format file. Error: {}. You can specify the structure manually", format_name, e.message()); + e.addMessage(fmt::format("Cannot extract table structure from {} format file. You can specify the structure manually", format_name)); + throw; } } else if (FormatFactory::instance().checkIfFormatHasSchemaReader(format_name)) @@ -85,6 +86,12 @@ ColumnsDescription readSchemaFromFormat( break; is_eof = buf->eof(); } + catch (Exception & e) + { + e.addMessage(fmt::format( + "Cannot extract table structure from {} format file. You can specify the structure manually", format_name)); + throw; + } catch (...) { auto exception_message = getCurrentExceptionMessage(false); @@ -136,7 +143,21 @@ ColumnsDescription readSchemaFromFormat( } if (!retry || !isRetryableSchemaInferenceError(getCurrentExceptionCode())) - throw Exception(ErrorCodes::CANNOT_EXTRACT_TABLE_STRUCTURE, "Cannot extract table structure from {} format file. Error: {}. You can specify the structure manually", format_name, exception_message); + { + try + { + throw; + } + catch (Exception & e) + { + e.addMessage(fmt::format("Cannot extract table structure from {} format file. You can specify the structure manually", format_name)); + throw; + } + catch (...) + { + throw Exception(ErrorCodes::CANNOT_EXTRACT_TABLE_STRUCTURE, "Cannot extract table structure from {} format file. Error: {}. You can specify the structure manually", format_name, exception_message); + } + } exception_messages += "\n" + exception_message; } diff --git a/src/Functions/CustomWeekTransforms.h b/src/Functions/CustomWeekTransforms.h index 3378aec02d5..5fa51d5f5e0 100644 --- a/src/Functions/CustomWeekTransforms.h +++ b/src/Functions/CustomWeekTransforms.h @@ -82,6 +82,14 @@ struct ToStartOfWeekImpl { return time_zone.toFirstDayNumOfWeek(DayNum(d), week_mode); } + static inline Int64 execute_extended_result(Int64 t, UInt8 week_mode, const DateLUTImpl & time_zone) + { + return time_zone.toFirstDayNumOfWeek(time_zone.toDayNum(t), week_mode); + } + static inline Int32 execute_extended_result(Int32 d, UInt8 week_mode, const DateLUTImpl & time_zone) + { + return time_zone.toFirstDayNumOfWeek(ExtendedDayNum(d), week_mode); + } using FactorTransform = ZeroTransform; }; @@ -115,7 +123,7 @@ struct ToWeekImpl using FactorTransform = ToStartOfYearImpl; }; -template +template struct WeekTransformer { explicit WeekTransformer(Transform transform_) @@ -130,7 +138,10 @@ struct WeekTransformer vec_to.resize(size); for (size_t i = 0; i < size; ++i) - vec_to[i] = transform.execute(vec_from[i], week_mode, time_zone); + if constexpr (is_extended_result) + vec_to[i] = transform.execute_extended_result(vec_from[i], week_mode, time_zone); + else + vec_to[i] = transform.execute(vec_from[i], week_mode, time_zone); } private: @@ -138,13 +149,13 @@ private: }; -template +template struct CustomWeekTransformImpl { template static ColumnPtr execute(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/, Transform transform = {}) { - const auto op = WeekTransformer{std::move(transform)}; + const auto op = WeekTransformer{std::move(transform)}; UInt8 week_mode = DEFAULT_WEEK_MODE; if (arguments.size() > 1) diff --git a/src/Functions/DateTimeTransforms.h b/src/Functions/DateTimeTransforms.h index 66d57f2463f..fbe8e4bfcfe 100644 --- a/src/Functions/DateTimeTransforms.h +++ b/src/Functions/DateTimeTransforms.h @@ -161,7 +161,14 @@ struct ToMondayImpl { return time_zone.toFirstDayNumOfWeek(DayNum(d)); } - + static inline Int64 execute_extended_result(Int64 t, const DateLUTImpl & time_zone) + { + return time_zone.toFirstDayNumOfWeek(time_zone.toDayNum(t)); + } + static inline Int32 execute_extended_result(Int32 d, const DateLUTImpl & time_zone) + { + return time_zone.toFirstDayNumOfWeek(ExtendedDayNum(d)); + } using FactorTransform = ZeroTransform; }; @@ -185,6 +192,14 @@ struct ToStartOfMonthImpl { return time_zone.toFirstDayNumOfMonth(DayNum(d)); } + static inline Int64 execute_extended_result(Int64 t, const DateLUTImpl & time_zone) + { + return time_zone.toFirstDayNumOfMonth(time_zone.toDayNum(t)); + } + static inline Int32 execute_extended_result(Int32 d, const DateLUTImpl & time_zone) + { + return time_zone.toFirstDayNumOfMonth(ExtendedDayNum(d)); + } using FactorTransform = ZeroTransform; }; @@ -218,7 +233,14 @@ struct ToLastDayOfMonthImpl /// 0xFFF9 is Int value for 2149-05-31 -- the last day where we can actually find LastDayOfMonth. This will also be the return value. return time_zone.toLastDayNumOfMonth(DayNum(std::min(d, UInt16(0xFFF9)))); } - + static inline Int64 execute_extended_result(Int64 t, const DateLUTImpl & time_zone) + { + return time_zone.toLastDayNumOfMonth(time_zone.toDayNum(t)); + } + static inline Int32 execute_extended_result(Int32 d, const DateLUTImpl & time_zone) + { + return time_zone.toLastDayNumOfMonth(ExtendedDayNum(d)); + } using FactorTransform = ZeroTransform; }; @@ -242,7 +264,14 @@ struct ToStartOfQuarterImpl { return time_zone.toFirstDayNumOfQuarter(DayNum(d)); } - + static inline Int64 execute_extended_result(Int64 t, const DateLUTImpl & time_zone) + { + return time_zone.toFirstDayNumOfQuarter(time_zone.toDayNum(t)); + } + static inline Int32 execute_extended_result(Int32 d, const DateLUTImpl & time_zone) + { + return time_zone.toFirstDayNumOfQuarter(ExtendedDayNum(d)); + } using FactorTransform = ZeroTransform; }; @@ -266,6 +295,14 @@ struct ToStartOfYearImpl { return time_zone.toFirstDayNumOfYear(DayNum(d)); } + static inline Int64 execute_extended_result(Int64 t, const DateLUTImpl & time_zone) + { + return time_zone.toFirstDayNumOfYear(time_zone.toDayNum(t)); + } + static inline Int32 execute_extended_result(Int32 d, const DateLUTImpl & time_zone) + { + return time_zone.toFirstDayNumOfYear(ExtendedDayNum(d)); + } using FactorTransform = ZeroTransform; }; @@ -893,7 +930,7 @@ struct ToStartOfISOYearImpl static inline UInt16 execute(Int64 t, const DateLUTImpl & time_zone) { - return time_zone.toFirstDayNumOfISOYear(time_zone.toDayNum(t)); + return t < 0 ? 0 : time_zone.toFirstDayNumOfISOYear(ExtendedDayNum(std::min(Int32(time_zone.toDayNum(t)), Int32(DATE_LUT_MAX_DAY_NUM)))); } static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone) { @@ -901,12 +938,20 @@ struct ToStartOfISOYearImpl } static inline UInt16 execute(Int32 d, const DateLUTImpl & time_zone) { - return time_zone.toFirstDayNumOfISOYear(ExtendedDayNum(d)); + return d < 0 ? 0 : time_zone.toFirstDayNumOfISOYear(ExtendedDayNum(std::min(d, Int32(DATE_LUT_MAX_DAY_NUM)))); } static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone) { return time_zone.toFirstDayNumOfISOYear(DayNum(d)); } + static inline Int64 execute_extended_result(Int64 t, const DateLUTImpl & time_zone) + { + return time_zone.toFirstDayNumOfISOYear(time_zone.toDayNum(t)); + } + static inline Int32 execute_extended_result(Int32 d, const DateLUTImpl & time_zone) + { + return time_zone.toFirstDayNumOfISOYear(ExtendedDayNum(d)); + } using FactorTransform = ZeroTransform; }; @@ -1201,7 +1246,7 @@ struct ToYYYYMMDDhhmmssImpl }; -template +template struct Transformer { template @@ -1211,18 +1256,21 @@ struct Transformer vec_to.resize(size); for (size_t i = 0; i < size; ++i) - vec_to[i] = transform.execute(vec_from[i], time_zone); + if constexpr (is_extended_result) + vec_to[i] = transform.execute_extended_result(vec_from[i], time_zone); + else + vec_to[i] = transform.execute(vec_from[i], time_zone); } }; -template +template struct DateTimeTransformImpl { static ColumnPtr execute( const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t /*input_rows_count*/, const Transform & transform = {}) { - using Op = Transformer; + using Op = Transformer; const ColumnPtr source_col = arguments[0].column; if (const auto * sources = checkAndGetColumn(source_col.get())) diff --git a/src/Functions/FunctionCustomWeekToDateOrDate32.h b/src/Functions/FunctionCustomWeekToDateOrDate32.h new file mode 100644 index 00000000000..0b91fbb3bbe --- /dev/null +++ b/src/Functions/FunctionCustomWeekToDateOrDate32.h @@ -0,0 +1,78 @@ +#pragma once +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int ILLEGAL_TYPE_OF_ARGUMENT; +} + +template +class FunctionCustomWeekToDateOrDate32 : public IFunctionCustomWeek, WithContext +{ +public: + const bool enable_extended_results_for_datetime_functions = false; + + static FunctionPtr create(ContextPtr context_) + { + return std::make_shared(context_); + } + + explicit FunctionCustomWeekToDateOrDate32(ContextPtr context_) + : WithContext(context_) + , enable_extended_results_for_datetime_functions(context_->getSettingsRef().enable_extended_results_for_datetime_functions) + { + } + + DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override + { + this->checkArguments(arguments, /*is_result_type_date_or_date32*/ true); + + const IDataType * from_type = arguments[0].type.get(); + WhichDataType which(from_type); + if ((which.isDate32() || which.isDateTime64()) && enable_extended_results_for_datetime_functions) + return std::make_shared(); + else + return std::make_shared(); + } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override + { + const IDataType * from_type = arguments[0].type.get(); + WhichDataType which(from_type); + + if (which.isDate()) + return CustomWeekTransformImpl::execute( + arguments, result_type, input_rows_count, Transform{}); + else if (which.isDate32()) + if (enable_extended_results_for_datetime_functions) + return CustomWeekTransformImpl::execute( + arguments, result_type, input_rows_count, Transform{}); + else + return CustomWeekTransformImpl::execute( + arguments, result_type, input_rows_count, Transform{}); + else if (which.isDateTime()) + return CustomWeekTransformImpl::execute( + arguments, result_type, input_rows_count, Transform{}); + else if (which.isDateTime64()) + { + if (enable_extended_results_for_datetime_functions) + return CustomWeekTransformImpl::execute( + arguments, result_type, input_rows_count, + TransformDateTime64{assert_cast(from_type)->getScale()}); + else + return CustomWeekTransformImpl::execute( + arguments, result_type, input_rows_count, + TransformDateTime64{assert_cast(from_type)->getScale()}); + } + else + throw Exception( + "Illegal type " + arguments[0].type->getName() + " of argument of function " + this->getName(), + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + } + +}; + +} diff --git a/src/Functions/FunctionCustomWeekToSomething.h b/src/Functions/FunctionCustomWeekToSomething.h index 8a0f474a7e8..eb65d562221 100644 --- a/src/Functions/FunctionCustomWeekToSomething.h +++ b/src/Functions/FunctionCustomWeekToSomething.h @@ -1,14 +1,5 @@ #pragma once -#include -#include -#include -#include -#include -#include -#include -#include -#include - +#include namespace DB { @@ -16,82 +7,23 @@ namespace DB namespace ErrorCodes { extern const int ILLEGAL_TYPE_OF_ARGUMENT; - extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; } /// See CustomWeekTransforms.h template -class FunctionCustomWeekToSomething : public IFunction +class FunctionCustomWeekToSomething : public IFunctionCustomWeek { public: - static constexpr auto name = Transform::name; static FunctionPtr create(ContextPtr) { return std::make_shared(); } - String getName() const override { return name; } - - bool isVariadic() const override { return true; } - bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; } - size_t getNumberOfArguments() const override { return 0; } - DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override { - if (arguments.size() == 1) - { - if (!isDate(arguments[0].type) && !isDate32(arguments[0].type) && !isDateTime(arguments[0].type) && !isDateTime64(arguments[0].type)) - throw Exception( - "Illegal type " + arguments[0].type->getName() + " of argument of function " + getName() - + ". Must be Date, Date32, DateTime or DateTime64.", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - } - else if (arguments.size() == 2) - { - if (!isDate(arguments[0].type) && !isDate32(arguments[0].type) && !isDateTime(arguments[0].type) && !isDateTime64(arguments[0].type)) - throw Exception( - "Illegal type " + arguments[0].type->getName() + " of 1st argument of function " + getName() - + ". Must be Date, Date32, DateTime or DateTime64.", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - if (!isUInt8(arguments[1].type)) - throw Exception( - "Illegal type of 2nd (optional) argument of function " + getName() - + ". Must be constant UInt8 (week mode).", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - } - else if (arguments.size() == 3) - { - if (!isDate(arguments[0].type) && !isDate32(arguments[0].type) && !isDateTime(arguments[0].type) && !isDateTime64(arguments[0].type)) - throw Exception( - "Illegal type " + arguments[0].type->getName() + " of argument of function " + getName() - + ". Must be Date, Date32, DateTime or DateTime64", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - if (!isUInt8(arguments[1].type)) - throw Exception( - "Illegal type of 2nd (optional) argument of function " + getName() - + ". Must be constant UInt8 (week mode).", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - if (!isString(arguments[2].type)) - throw Exception( - "Illegal type of 3rd (optional) argument of function " + getName() - + ". Must be constant string (timezone name).", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - if ((isDate(arguments[0].type) || isDate32(arguments[0].type)) - && (std::is_same_v || std::is_same_v)) - throw Exception( - "The timezone argument of function " + getName() + " is allowed only when the 1st argument is DateTime or DateTime64.", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - } - else - throw Exception( - "Number of arguments for function " + getName() + " doesn't match: passed " + toString(arguments.size()) - + ", expected 1, 2 or 3.", - ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + this->checkArguments(arguments); return std::make_shared(); } - bool useDefaultImplementationForConstants() const override { return true; } - ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1, 2}; } - ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override { const IDataType * from_type = arguments[0].type.get(); @@ -114,44 +46,10 @@ public: } else throw Exception( - "Illegal type " + arguments[0].type->getName() + " of argument of function " + getName(), + "Illegal type " + arguments[0].type->getName() + " of argument of function " + this->getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); } - - bool hasInformationAboutMonotonicity() const override { return true; } - - Monotonicity getMonotonicityForRange(const IDataType & type, const Field & left, const Field & right) const override - { - if constexpr (std::is_same_v) - return { .is_monotonic = true, .is_always_monotonic = true }; - - const IFunction::Monotonicity is_monotonic = { .is_monotonic = true }; - const IFunction::Monotonicity is_not_monotonic; - - /// This method is called only if the function has one argument. Therefore, we do not care about the non-local time zone. - const DateLUTImpl & date_lut = DateLUT::instance(); - - if (left.isNull() || right.isNull()) - return {}; - - /// The function is monotonous on the [left, right] segment, if the factor transformation returns the same values for them. - - if (checkAndGetDataType(&type)) - { - return Transform::FactorTransform::execute(UInt16(left.get()), date_lut) - == Transform::FactorTransform::execute(UInt16(right.get()), date_lut) - ? is_monotonic - : is_not_monotonic; - } - else - { - return Transform::FactorTransform::execute(UInt32(left.get()), date_lut) - == Transform::FactorTransform::execute(UInt32(right.get()), date_lut) - ? is_monotonic - : is_not_monotonic; - } - } }; } diff --git a/src/Functions/FunctionDateOrDateTimeToDateOrDate32.h b/src/Functions/FunctionDateOrDateTimeToDateOrDate32.h new file mode 100644 index 00000000000..3ff90cb57fb --- /dev/null +++ b/src/Functions/FunctionDateOrDateTimeToDateOrDate32.h @@ -0,0 +1,81 @@ +#pragma once +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int ILLEGAL_TYPE_OF_ARGUMENT; +} + +template +class FunctionDateOrDateTimeToDateOrDate32 : public IFunctionDateOrDateTime, WithContext +{ +public: + const bool enable_extended_results_for_datetime_functions = false; + + static FunctionPtr create(ContextPtr context_) + { + return std::make_shared(context_); + } + + explicit FunctionDateOrDateTimeToDateOrDate32(ContextPtr context_) + : WithContext(context_) + , enable_extended_results_for_datetime_functions(context_->getSettingsRef().enable_extended_results_for_datetime_functions) + { + } + + DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override + { + this->checkArguments(arguments, /*is_result_type_date_or_date32*/ true); + + const IDataType * from_type = arguments[0].type.get(); + WhichDataType which(from_type); + + /// If the time zone is specified but empty, throw an exception. + /// only validate the time_zone part if the number of arguments is 2. + if ((which.isDateTime() || which.isDateTime64()) && arguments.size() == 2 + && extractTimeZoneNameFromFunctionArguments(arguments, 1, 0).empty()) + throw Exception( + "Function " + this->getName() + " supports a 2nd argument (optional) that must be non-empty and be a valid time zone", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + if ((which.isDate32() || which.isDateTime64()) && enable_extended_results_for_datetime_functions) + return std::make_shared(); + else + return std::make_shared(); + } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override + { + const IDataType * from_type = arguments[0].type.get(); + WhichDataType which(from_type); + + if (which.isDate()) + return DateTimeTransformImpl::execute(arguments, result_type, input_rows_count); + else if (which.isDate32()) + if (enable_extended_results_for_datetime_functions) + return DateTimeTransformImpl::execute(arguments, result_type, input_rows_count); + else + return DateTimeTransformImpl::execute(arguments, result_type, input_rows_count); + else if (which.isDateTime()) + return DateTimeTransformImpl::execute(arguments, result_type, input_rows_count); + else if (which.isDateTime64()) + { + const auto scale = static_cast(from_type)->getScale(); + + const TransformDateTime64 transformer(scale); + if (enable_extended_results_for_datetime_functions) + return DateTimeTransformImpl::execute(arguments, result_type, input_rows_count, transformer); + else + return DateTimeTransformImpl::execute(arguments, result_type, input_rows_count, transformer); + } + else + throw Exception("Illegal type " + arguments[0].type->getName() + " of argument of function " + this->getName(), + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + } + +}; + +} diff --git a/src/Functions/FunctionDateOrDateTimeToSomething.h b/src/Functions/FunctionDateOrDateTimeToSomething.h index d734c7f87c1..5c1c54c1b84 100644 --- a/src/Functions/FunctionDateOrDateTimeToSomething.h +++ b/src/Functions/FunctionDateOrDateTimeToSomething.h @@ -1,14 +1,5 @@ #pragma once -#include -#include -#include -#include -#include -#include -#include -#include -#include - +#include namespace DB { @@ -16,59 +7,18 @@ namespace DB namespace ErrorCodes { extern const int ILLEGAL_TYPE_OF_ARGUMENT; - extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; } /// See DateTimeTransforms.h template -class FunctionDateOrDateTimeToSomething : public IFunction +class FunctionDateOrDateTimeToSomething : public IFunctionDateOrDateTime { public: - static constexpr auto name = Transform::name; static FunctionPtr create(ContextPtr) { return std::make_shared(); } - String getName() const override - { - return name; - } - - bool isVariadic() const override { return true; } - bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return false; } - size_t getNumberOfArguments() const override { return 0; } - DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override { - if (arguments.size() == 1) - { - if (!isDateOrDate32(arguments[0].type) && !isDateTime(arguments[0].type) && !isDateTime64(arguments[0].type)) - throw Exception( - "Illegal type " + arguments[0].type->getName() + " of argument of function " + getName() - + ". Should be a date or a date with time", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - } - else if (arguments.size() == 2) - { - if (!isDateOrDate32(arguments[0].type) && !isDateTime(arguments[0].type) && !isDateTime64(arguments[0].type)) - throw Exception( - "Illegal type " + arguments[0].type->getName() + " of argument of function " + getName() - + ". Should be a date or a date with time", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - if (!isString(arguments[1].type)) - throw Exception( - "Function " + getName() + " supports 1 or 2 arguments. The 1st argument " - "must be of type Date or DateTime. The 2nd argument (optional) must be " - "a constant string with timezone name", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - if ((isDate(arguments[0].type) || isDate32(arguments[0].type)) && (std::is_same_v || std::is_same_v)) - throw Exception( - "The timezone argument of function " + getName() + " is allowed only when the 1st argument has the type DateTime", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - } - else - throw Exception( - "Number of arguments for function " + getName() + " doesn't match: passed " + toString(arguments.size()) - + ", should be 1 or 2", - ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + this->checkArguments(arguments, (std::is_same_v || std::is_same_v)); /// For DateTime, if time zone is specified, attach it to type. /// If the time zone is specified but empty, throw an exception. @@ -79,7 +29,7 @@ public: /// to accommodate functions like toStartOfDay(today()), toStartOfDay(yesterday()) etc. if (arguments.size() == 2 && time_zone.empty()) throw Exception( - "Function " + getName() + " supports a 2nd argument (optional) that must be non-empty and be a valid time zone", + "Function " + this->getName() + " supports a 2nd argument (optional) that must be non-empty and be a valid time zone", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); return std::make_shared(time_zone); } @@ -109,9 +59,6 @@ public: return std::make_shared(); } - bool useDefaultImplementationForConstants() const override { return true; } - ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1}; } - ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override { const IDataType * from_type = arguments[0].type.get(); @@ -131,51 +78,10 @@ public: return DateTimeTransformImpl::execute(arguments, result_type, input_rows_count, transformer); } else - throw Exception("Illegal type " + arguments[0].type->getName() + " of argument of function " + getName(), + throw Exception("Illegal type " + arguments[0].type->getName() + " of argument of function " + this->getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); } - bool hasInformationAboutMonotonicity() const override - { - return true; - } - - Monotonicity getMonotonicityForRange(const IDataType & type, const Field & left, const Field & right) const override - { - if constexpr (std::is_same_v) - return { .is_monotonic = true, .is_always_monotonic = true }; - - const IFunction::Monotonicity is_monotonic = { .is_monotonic = true }; - const IFunction::Monotonicity is_not_monotonic; - - const DateLUTImpl * date_lut = &DateLUT::instance(); - if (const auto * timezone = dynamic_cast(&type)) - date_lut = &timezone->getTimeZone(); - - if (left.isNull() || right.isNull()) - return is_not_monotonic; - - /// The function is monotonous on the [left, right] segment, if the factor transformation returns the same values for them. - - if (checkAndGetDataType(&type)) - { - return Transform::FactorTransform::execute(UInt16(left.get()), *date_lut) - == Transform::FactorTransform::execute(UInt16(right.get()), *date_lut) - ? is_monotonic : is_not_monotonic; - } - else if (checkAndGetDataType(&type)) - { - return Transform::FactorTransform::execute(Int32(left.get()), *date_lut) - == Transform::FactorTransform::execute(Int32(right.get()), *date_lut) - ? is_monotonic : is_not_monotonic; - } - else - { - return Transform::FactorTransform::execute(UInt32(left.get()), *date_lut) - == Transform::FactorTransform::execute(UInt32(right.get()), *date_lut) - ? is_monotonic : is_not_monotonic; - } - } }; } diff --git a/src/Functions/FunctionsConversion.h b/src/Functions/FunctionsConversion.h index 96c28b21ef0..a8d2882c653 100644 --- a/src/Functions/FunctionsConversion.h +++ b/src/Functions/FunctionsConversion.h @@ -2381,7 +2381,7 @@ using FunctionToDate = FunctionConvert; using FunctionToDateTime = FunctionConvert; using FunctionToDateTime32 = FunctionConvert; -using FunctionToDateTime64 = FunctionConvert; +using FunctionToDateTime64 = FunctionConvert; using FunctionToUUID = FunctionConvert>; using FunctionToString = FunctionConvert; using FunctionToUnixTimestamp = FunctionConvert>; diff --git a/src/Functions/FunctionsEmbeddedDictionaries.h b/src/Functions/FunctionsEmbeddedDictionaries.h index 20be3ee3cce..aa2144d271f 100644 --- a/src/Functions/FunctionsEmbeddedDictionaries.h +++ b/src/Functions/FunctionsEmbeddedDictionaries.h @@ -649,7 +649,7 @@ public: for (unsigned int region_id : region_ids) { const StringRef & name_ref = dict.getRegionName(region_id, language); - col_to->insertDataWithTerminatingZero(name_ref.data, name_ref.size + 1); + col_to->insertData(name_ref.data, name_ref.size); } return col_to; diff --git a/src/Functions/GatherUtils/Sources.h b/src/Functions/GatherUtils/Sources.h index 13e3de99552..b78e70975c6 100644 --- a/src/Functions/GatherUtils/Sources.h +++ b/src/Functions/GatherUtils/Sources.h @@ -140,17 +140,12 @@ struct NumericArraySource : public ArraySourceImpl> /// The methods can be virtual or not depending on the template parameter. See IStringSource. -#if !defined(__clang__) -# pragma GCC diagnostic push +#pragma GCC diagnostic push +#ifdef HAS_SUGGEST_OVERRIDE # pragma GCC diagnostic ignored "-Wsuggest-override" -#elif __clang_major__ >= 11 -# pragma GCC diagnostic push -# ifdef HAS_SUGGEST_OVERRIDE -# pragma GCC diagnostic ignored "-Wsuggest-override" -# endif -# ifdef HAS_SUGGEST_DESTRUCTOR_OVERRIDE -# pragma GCC diagnostic ignored "-Wsuggest-destructor-override" -# endif +#endif +#ifdef HAS_SUGGEST_DESTRUCTOR_OVERRIDE +# pragma GCC diagnostic ignored "-Wsuggest-destructor-override" #endif template @@ -233,9 +228,7 @@ struct ConstSource : public Base } }; -#if !defined(__clang__) || __clang_major__ >= 11 -# pragma GCC diagnostic pop -#endif +#pragma GCC diagnostic pop struct StringSource { diff --git a/src/Functions/IFunctionCustomWeek.h b/src/Functions/IFunctionCustomWeek.h new file mode 100644 index 00000000000..1bc4e44655a --- /dev/null +++ b/src/Functions/IFunctionCustomWeek.h @@ -0,0 +1,122 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; +} + +template +class IFunctionCustomWeek : public IFunction +{ +public: + static constexpr auto name = Transform::name; + String getName() const override { return name; } + bool isVariadic() const override { return true; } + bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; } + size_t getNumberOfArguments() const override { return 0; } + bool useDefaultImplementationForConstants() const override { return true; } + ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1, 2}; } + + bool hasInformationAboutMonotonicity() const override { return true; } + + Monotonicity getMonotonicityForRange(const IDataType & type, const Field & left, const Field & right) const override + { + if constexpr (std::is_same_v) + return {.is_monotonic = true, .is_always_monotonic = true}; + + const IFunction::Monotonicity is_monotonic = {.is_monotonic = true}; + const IFunction::Monotonicity is_not_monotonic; + + /// This method is called only if the function has one argument. Therefore, we do not care about the non-local time zone. + const DateLUTImpl & date_lut = DateLUT::instance(); + + if (left.isNull() || right.isNull()) + return {}; + + /// The function is monotonous on the [left, right] segment, if the factor transformation returns the same values for them. + + if (checkAndGetDataType(&type)) + { + return Transform::FactorTransform::execute(UInt16(left.get()), date_lut) + == Transform::FactorTransform::execute(UInt16(right.get()), date_lut) + ? is_monotonic + : is_not_monotonic; + } + else + { + return Transform::FactorTransform::execute(UInt32(left.get()), date_lut) + == Transform::FactorTransform::execute(UInt32(right.get()), date_lut) + ? is_monotonic + : is_not_monotonic; + } + } + +protected: + void checkArguments(const ColumnsWithTypeAndName & arguments, bool is_result_type_date_or_date32 = false) const + { + if (arguments.size() == 1) + { + if (!isDate(arguments[0].type) && !isDate32(arguments[0].type) && !isDateTime(arguments[0].type) && !isDateTime64(arguments[0].type)) + throw Exception( + "Illegal type " + arguments[0].type->getName() + " of argument of function " + getName() + + ". Must be Date, Date32, DateTime or DateTime64.", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + } + else if (arguments.size() == 2) + { + if (!isDate(arguments[0].type) && !isDate32(arguments[0].type) && !isDateTime(arguments[0].type) && !isDateTime64(arguments[0].type)) + throw Exception( + "Illegal type " + arguments[0].type->getName() + " of 1st argument of function " + getName() + + ". Must be Date, Date32, DateTime or DateTime64.", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + if (!isUInt8(arguments[1].type)) + throw Exception( + "Illegal type of 2nd (optional) argument of function " + getName() + + ". Must be constant UInt8 (week mode).", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + } + else if (arguments.size() == 3) + { + if (!isDate(arguments[0].type) && !isDate32(arguments[0].type) && !isDateTime(arguments[0].type) && !isDateTime64(arguments[0].type)) + throw Exception( + "Illegal type " + arguments[0].type->getName() + " of argument of function " + getName() + + ". Must be Date, Date32, DateTime or DateTime64", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + if (!isUInt8(arguments[1].type)) + throw Exception( + "Illegal type of 2nd (optional) argument of function " + getName() + + ". Must be constant UInt8 (week mode).", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + if (!isString(arguments[2].type)) + throw Exception( + "Illegal type of 3rd (optional) argument of function " + getName() + + ". Must be constant string (timezone name).", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + if ((isDate(arguments[0].type) || isDate32(arguments[0].type)) && is_result_type_date_or_date32) + throw Exception( + "The timezone argument of function " + getName() + " is allowed only when the 1st argument is DateTime or DateTime64.", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + } + else + throw Exception( + "Number of arguments for function " + getName() + " doesn't match: passed " + toString(arguments.size()) + + ", expected 1, 2 or 3.", + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + } + +}; + +} diff --git a/src/Functions/IFunctionDateOrDateTime.h b/src/Functions/IFunctionDateOrDateTime.h new file mode 100644 index 00000000000..1efe89c7fe9 --- /dev/null +++ b/src/Functions/IFunctionDateOrDateTime.h @@ -0,0 +1,118 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; +} + +template +class IFunctionDateOrDateTime : public IFunction +{ +public: + static constexpr auto name = Transform::name; + String getName() const override { return name; } + + bool isVariadic() const override { return true; } + + bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return false; } + + size_t getNumberOfArguments() const override { return 0; } + + bool useDefaultImplementationForConstants() const override { return true; } + + ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1}; } + + bool hasInformationAboutMonotonicity() const override + { + return true; + } + + Monotonicity getMonotonicityForRange(const IDataType & type, const Field & left, const Field & right) const override + { + if constexpr (std::is_same_v) + return { .is_monotonic = true, .is_always_monotonic = true }; + + const IFunction::Monotonicity is_monotonic = { .is_monotonic = true }; + const IFunction::Monotonicity is_not_monotonic; + + const DateLUTImpl * date_lut = &DateLUT::instance(); + if (const auto * timezone = dynamic_cast(&type)) + date_lut = &timezone->getTimeZone(); + + if (left.isNull() || right.isNull()) + return is_not_monotonic; + + /// The function is monotonous on the [left, right] segment, if the factor transformation returns the same values for them. + + if (checkAndGetDataType(&type)) + { + return Transform::FactorTransform::execute(UInt16(left.get()), *date_lut) + == Transform::FactorTransform::execute(UInt16(right.get()), *date_lut) + ? is_monotonic : is_not_monotonic; + } + else if (checkAndGetDataType(&type)) + { + return Transform::FactorTransform::execute(Int32(left.get()), *date_lut) + == Transform::FactorTransform::execute(Int32(right.get()), *date_lut) + ? is_monotonic : is_not_monotonic; + } + else + { + return Transform::FactorTransform::execute(UInt32(left.get()), *date_lut) + == Transform::FactorTransform::execute(UInt32(right.get()), *date_lut) + ? is_monotonic : is_not_monotonic; + } + } + +protected: + void checkArguments(const ColumnsWithTypeAndName & arguments, bool is_result_type_date_or_date32) const + { + if (arguments.size() == 1) + { + if (!isDateOrDate32(arguments[0].type) && !isDateTime(arguments[0].type) && !isDateTime64(arguments[0].type)) + throw Exception( + "Illegal type " + arguments[0].type->getName() + " of argument of function " + getName() + + ". Should be Date, Date32, DateTime or DateTime64", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + } + else if (arguments.size() == 2) + { + if (!isDateOrDate32(arguments[0].type) && !isDateTime(arguments[0].type) && !isDateTime64(arguments[0].type)) + throw Exception( + "Illegal type " + arguments[0].type->getName() + " of argument of function " + getName() + + ". Should be Date, Date32, DateTime or DateTime64", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + if (!isString(arguments[1].type)) + throw Exception( + "Function " + getName() + " supports 1 or 2 arguments. The optional 2nd argument must be " + "a constant string with a timezone name", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + if ((isDate(arguments[0].type) || isDate32(arguments[0].type)) && is_result_type_date_or_date32) + throw Exception( + "The timezone argument of function " + getName() + " is allowed only when the 1st argument has the type DateTime or DateTime64", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + } + else + throw Exception( + "Number of arguments for function " + getName() + " doesn't match: passed " + toString(arguments.size()) + + ", should be 1 or 2", + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + } +}; + +} diff --git a/src/Functions/PolygonUtils.h b/src/Functions/PolygonUtils.h index de4bb2d48de..c16db1d1ff7 100644 --- a/src/Functions/PolygonUtils.h +++ b/src/Functions/PolygonUtils.h @@ -12,11 +12,6 @@ /// Warning in boost::geometry during template strategy substitution. #pragma GCC diagnostic push - -#if !defined(__clang__) -#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" -#endif - #pragma GCC diagnostic ignored "-Wunused-parameter" #include @@ -286,16 +281,9 @@ void PointInPolygonWithGrid::calcGridAttributes( const Point & min_corner = box.min_corner(); const Point & max_corner = box.max_corner(); -#pragma GCC diagnostic push -#if !defined(__clang__) -#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" -#endif - cell_width = (max_corner.x() - min_corner.x()) / grid_size; cell_height = (max_corner.y() - min_corner.y()) / grid_size; -#pragma GCC diagnostic pop - if (cell_width == 0 || cell_height == 0) { has_empty_bound = true; @@ -330,10 +318,6 @@ void PointInPolygonWithGrid::buildGrid() for (size_t row = 0; row < grid_size; ++row) { -#pragma GCC diagnostic push -#if !defined(__clang__) -#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" -#endif CoordinateType y_min = min_corner.y() + row * cell_height; CoordinateType y_max = min_corner.y() + (row + 1) * cell_height; @@ -341,7 +325,6 @@ void PointInPolygonWithGrid::buildGrid() { CoordinateType x_min = min_corner.x() + col * cell_width; CoordinateType x_max = min_corner.x() + (col + 1) * cell_width; -#pragma GCC diagnostic pop Box cell_box(Point(x_min, y_min), Point(x_max, y_max)); MultiPolygon intersection; diff --git a/src/Functions/TransformDateTime64.h b/src/Functions/TransformDateTime64.h index 9ac28118b8f..fbe7e2e8250 100644 --- a/src/Functions/TransformDateTime64.h +++ b/src/Functions/TransformDateTime64.h @@ -87,6 +87,46 @@ public: return wrapped_transform.execute(t, std::forward(args)...); } + + template + inline auto NO_SANITIZE_UNDEFINED execute_extended_result(const DateTime64 & t, Args && ... args) const + { + /// Type conversion from float to integer may be required. + /// We are Ok with implementation specific result for out of range and denormals conversion. + + if constexpr (TransformHasExecuteOverload_v) + { + return wrapped_transform.execute_extended_result(t, scale_multiplier, std::forward(args)...); + } + else if constexpr (TransformHasExecuteOverload_v, Args...>) + { + auto components = DecimalUtils::splitWithScaleMultiplier(t, scale_multiplier); + + const auto result = wrapped_transform.execute_extended_result(components, std::forward(args)...); + using ResultType = std::decay_t; + + if constexpr (std::is_same_v, ResultType>) + { + return DecimalUtils::decimalFromComponentsWithMultiplier(result, scale_multiplier); + } + else + { + return result; + } + } + else + { + const auto components = DecimalUtils::splitWithScaleMultiplier(t, scale_multiplier); + return wrapped_transform.execute_extended_result(static_cast(components.whole), std::forward(args)...); + } + } + + template >> + inline auto execute_extended_result(const T & t, Args && ... args) const + { + return wrapped_transform.execute_extended_result(t, std::forward(args)...); + } + private: DateTime64::NativeType scale_multiplier = 1; Transform wrapped_transform = {}; diff --git a/src/Functions/addressToSymbol.cpp b/src/Functions/addressToSymbol.cpp index 99988ee82f6..dd9efd6cc44 100644 --- a/src/Functions/addressToSymbol.cpp +++ b/src/Functions/addressToSymbol.cpp @@ -83,7 +83,7 @@ public: for (size_t i = 0; i < input_rows_count; ++i) { if (const auto * symbol = symbol_index.findSymbol(reinterpret_cast(data[i]))) - result_column->insertDataWithTerminatingZero(symbol->name, strlen(symbol->name) + 1); + result_column->insertData(symbol->name, strlen(symbol->name)); else result_column->insertDefault(); } diff --git a/src/Functions/modelEvaluate.cpp b/src/Functions/catboostEvaluate.cpp similarity index 55% rename from src/Functions/modelEvaluate.cpp rename to src/Functions/catboostEvaluate.cpp index 3ee2ae3fae4..1ac7815239e 100644 --- a/src/Functions/modelEvaluate.cpp +++ b/src/Functions/catboostEvaluate.cpp @@ -1,18 +1,18 @@ #include #include -#include -#include -#include -#include -#include -#include -#include +#include +#include #include +#include #include -#include +#include #include +#include +#include +#include #include +#include #include @@ -21,66 +21,80 @@ namespace DB namespace ErrorCodes { + extern const int FILE_DOESNT_EXIST; extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int TOO_FEW_ARGUMENTS_FOR_FUNCTION; extern const int ILLEGAL_COLUMN; } -class ExternalModelsLoader; - - -/// Evaluate external model. -/// First argument - model name, the others - model arguments. -/// * for CatBoost model - float features first, then categorical -/// Result - Float64. -class FunctionModelEvaluate final : public IFunction +/// Evaluate CatBoost model. +/// - Arguments: float features first, then categorical features. +/// - Result: Float64. +class FunctionCatBoostEvaluate final : public IFunction, WithContext { +private: + mutable std::unique_ptr bridge_helper; + public: - static constexpr auto name = "modelEvaluate"; + static constexpr auto name = "catboostEvaluate"; - static FunctionPtr create(ContextPtr context) - { - return std::make_shared(context->getExternalModelsLoader()); - } - - explicit FunctionModelEvaluate(const ExternalModelsLoader & models_loader_) - : models_loader(models_loader_) {} + static FunctionPtr create(ContextPtr context_) { return std::make_shared(context_); } + explicit FunctionCatBoostEvaluate(ContextPtr context_) : WithContext(context_) {} String getName() const override { return name; } - bool isVariadic() const override { return true; } - bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; } - bool isDeterministic() const override { return false; } - bool useDefaultImplementationForNulls() const override { return false; } - size_t getNumberOfArguments() const override { return 0; } + void initBridge(const ColumnConst * name_col) const + { + String library_path = getContext()->getConfigRef().getString("catboost_lib_path"); + if (!std::filesystem::exists(library_path)) + throw Exception(ErrorCodes::FILE_DOESNT_EXIST, "Can't load library {}: file doesn't exist", library_path); + + String model_path = name_col->getValue(); + if (!std::filesystem::exists(model_path)) + throw Exception(ErrorCodes::FILE_DOESNT_EXIST, "Can't load model {}: file doesn't exist", model_path); + + bridge_helper = std::make_unique(getContext(), model_path, library_path); + } + + DataTypePtr getReturnTypeFromLibraryBridge() const + { + size_t tree_count = bridge_helper->getTreeCount(); + auto type = std::make_shared(); + if (tree_count == 1) + return type; + + DataTypes types(tree_count, type); + + return std::make_shared(types); + } DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override { if (arguments.size() < 2) - throw Exception("Function " + getName() + " expects at least 2 arguments", - ErrorCodes::TOO_FEW_ARGUMENTS_FOR_FUNCTION); + throw Exception(ErrorCodes::TOO_FEW_ARGUMENTS_FOR_FUNCTION, "Function {} expects at least 2 arguments", getName()); if (!isString(arguments[0].type)) - throw Exception("Illegal type " + arguments[0].type->getName() + " of first argument of function " + getName() - + ", expected a string.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + throw Exception( + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Illegal type {} of first argument of function {}, expected a string.", arguments[0].type->getName(), getName()); const auto * name_col = checkAndGetColumnConst(arguments[0].column.get()); if (!name_col) - throw Exception("First argument of function " + getName() + " must be a constant string", - ErrorCodes::ILLEGAL_COLUMN); + throw Exception(ErrorCodes::ILLEGAL_COLUMN, "First argument of function {} must be a constant string", getName()); + + initBridge(name_col); + + auto type = getReturnTypeFromLibraryBridge(); bool has_nullable = false; for (size_t i = 1; i < arguments.size(); ++i) has_nullable = has_nullable || arguments[i].type->isNullable(); - auto model = models_loader.getModel(name_col->getValue()); - auto type = model->getReturnType(); - if (has_nullable) { if (const auto * tuple = typeid_cast(type.get())) @@ -98,31 +112,25 @@ public: return type; } - ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t) const override { const auto * name_col = checkAndGetColumnConst(arguments[0].column.get()); if (!name_col) - throw Exception("First argument of function " + getName() + " must be a constant string", - ErrorCodes::ILLEGAL_COLUMN); - - auto model = models_loader.getModel(name_col->getValue()); + throw Exception(ErrorCodes::ILLEGAL_COLUMN, "First argument of function {} must be a constant string", getName()); ColumnRawPtrs column_ptrs; Columns materialized_columns; ColumnPtr null_map; - column_ptrs.reserve(arguments.size()); - for (auto arg : collections::range(1, arguments.size())) + ColumnsWithTypeAndName feature_arguments(arguments.begin() + 1, arguments.end()); + for (auto & arg : feature_arguments) { - const auto & column = arguments[arg].column; - column_ptrs.push_back(column.get()); - if (auto full_column = column->convertToFullColumnIfConst()) + if (auto full_column = arg.column->convertToFullColumnIfConst()) { materialized_columns.push_back(full_column); - column_ptrs.back() = full_column.get(); + arg.column = full_column; } - if (const auto * col_nullable = checkAndGetColumn(*column_ptrs.back())) + if (const auto * col_nullable = checkAndGetColumn(&*arg.column)) { if (!null_map) null_map = col_nullable->getNullMapColumnPtr(); @@ -140,11 +148,12 @@ public: null_map = std::move(mut_null_map); } - column_ptrs.back() = &col_nullable->getNestedColumn(); + arg.column = col_nullable->getNestedColumn().getPtr(); + arg.type = static_cast(*arg.type).getNestedType(); } } - auto res = model->evaluate(column_ptrs); + auto res = bridge_helper->evaluate(feature_arguments); if (null_map) { @@ -162,15 +171,12 @@ public: return res; } - -private: - const ExternalModelsLoader & models_loader; }; -REGISTER_FUNCTION(ExternalModels) +REGISTER_FUNCTION(CatBoostEvaluate) { - factory.registerFunction(); + factory.registerFunction(); } } diff --git a/src/Functions/demange.cpp b/src/Functions/demange.cpp index 9026790f740..a7c3d8e52bf 100644 --- a/src/Functions/demange.cpp +++ b/src/Functions/demange.cpp @@ -78,15 +78,15 @@ public: for (size_t i = 0; i < input_rows_count; ++i) { - StringRef source = column_concrete->getDataAtWithTerminatingZero(i); + StringRef source = column_concrete->getDataAt(i); auto demangled = tryDemangle(source.data); if (demangled) { - result_column->insertDataWithTerminatingZero(demangled.get(), strlen(demangled.get()) + 1); + result_column->insertData(demangled.get(), strlen(demangled.get())); } else { - result_column->insertDataWithTerminatingZero(source.data, source.size); + result_column->insertData(source.data, source.size); } } @@ -102,4 +102,3 @@ REGISTER_FUNCTION(Demangle) } } - diff --git a/src/Functions/grouping.h b/src/Functions/grouping.h index a49e946b2cb..b9ef6ffc107 100644 --- a/src/Functions/grouping.h +++ b/src/Functions/grouping.h @@ -1,9 +1,9 @@ #pragma once +#include #include #include #include -#include #include #include #include @@ -19,10 +19,17 @@ protected: static constexpr UInt64 ONE = 1; const ColumnNumbers arguments_indexes; + // Initial implementation of GROUPING function returned 1 if the argument is used as an aggregation key. + // This differs from the behavior described in the standard and other DBMS. + const bool force_compatibility; + + static constexpr UInt64 COMPATIBLE_MODE[] = {1, 0}; + static constexpr UInt64 INCOMPATIBLE_MODE[] = {0, 1}; public: - FunctionGroupingBase(ColumnNumbers arguments_indexes_) + FunctionGroupingBase(ColumnNumbers arguments_indexes_, bool force_compatibility_) : arguments_indexes(std::move(arguments_indexes_)) + , force_compatibility(force_compatibility_) {} bool isVariadic() const override { return true; } @@ -48,13 +55,15 @@ public: auto result = ColumnUInt64::create(); auto & result_data = result->getData(); result_data.reserve(input_rows_count); + + const auto * result_table = likely(force_compatibility) ? COMPATIBLE_MODE : INCOMPATIBLE_MODE; for (size_t i = 0; i < input_rows_count; ++i) { UInt64 set_index = grouping_set_column->getElement(i); UInt64 value = 0; for (auto index : arguments_indexes) - value = (value << 1) + (checker(set_index, index) ? 1 : 0); + value = (value << 1) + result_table[checker(set_index, index) ? 1 : 0]; result_data.push_back(value); } @@ -65,14 +74,16 @@ public: class FunctionGroupingOrdinary : public FunctionGroupingBase { public: - explicit FunctionGroupingOrdinary(ColumnNumbers arguments_indexes_) - : FunctionGroupingBase(std::move(arguments_indexes_)) + FunctionGroupingOrdinary(ColumnNumbers arguments_indexes_, bool force_compatibility_) + : FunctionGroupingBase(std::move(arguments_indexes_), force_compatibility_) {} String getName() const override { return "groupingOrdinary"; } ColumnPtr executeImpl(const ColumnsWithTypeAndName &, const DataTypePtr &, size_t input_rows_count) const override { + if (likely(force_compatibility)) + return ColumnUInt64::create(input_rows_count, 0); UInt64 value = (ONE << arguments_indexes.size()) - 1; return ColumnUInt64::create(input_rows_count, value); } @@ -83,8 +94,8 @@ class FunctionGroupingForRollup : public FunctionGroupingBase const UInt64 aggregation_keys_number; public: - FunctionGroupingForRollup(ColumnNumbers arguments_indexes_, UInt64 aggregation_keys_number_) - : FunctionGroupingBase(std::move(arguments_indexes_)) + FunctionGroupingForRollup(ColumnNumbers arguments_indexes_, UInt64 aggregation_keys_number_, bool force_compatibility_) + : FunctionGroupingBase(std::move(arguments_indexes_), force_compatibility_) , aggregation_keys_number(aggregation_keys_number_) {} @@ -113,8 +124,8 @@ class FunctionGroupingForCube : public FunctionGroupingBase public: - FunctionGroupingForCube(ColumnNumbers arguments_indexes_, UInt64 aggregation_keys_number_) - : FunctionGroupingBase(arguments_indexes_) + FunctionGroupingForCube(ColumnNumbers arguments_indexes_, UInt64 aggregation_keys_number_, bool force_compatibility_) + : FunctionGroupingBase(arguments_indexes_, force_compatibility_) , aggregation_keys_number(aggregation_keys_number_) {} @@ -142,8 +153,8 @@ class FunctionGroupingForGroupingSets : public FunctionGroupingBase { ColumnNumbersSetList grouping_sets; public: - FunctionGroupingForGroupingSets(ColumnNumbers arguments_indexes_, ColumnNumbersList const & grouping_sets_) - : FunctionGroupingBase(std::move(arguments_indexes_)) + FunctionGroupingForGroupingSets(ColumnNumbers arguments_indexes_, ColumnNumbersList const & grouping_sets_, bool force_compatibility_) + : FunctionGroupingBase(std::move(arguments_indexes_), force_compatibility_) { for (auto const & set : grouping_sets_) grouping_sets.emplace_back(set.begin(), set.end()); diff --git a/src/Functions/toCustomWeek.cpp b/src/Functions/toCustomWeek.cpp index 13dc76b6389..b773cc7df96 100644 --- a/src/Functions/toCustomWeek.cpp +++ b/src/Functions/toCustomWeek.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include @@ -9,7 +10,7 @@ namespace DB { using FunctionToWeek = FunctionCustomWeekToSomething; using FunctionToYearWeek = FunctionCustomWeekToSomething; -using FunctionToStartOfWeek = FunctionCustomWeekToSomething; +using FunctionToStartOfWeek = FunctionCustomWeekToDateOrDate32; REGISTER_FUNCTION(ToCustomWeek) { diff --git a/src/Functions/toLastDayOfMonth.cpp b/src/Functions/toLastDayOfMonth.cpp index a7faab15f9f..9365880bfb8 100644 --- a/src/Functions/toLastDayOfMonth.cpp +++ b/src/Functions/toLastDayOfMonth.cpp @@ -1,12 +1,12 @@ #include #include -#include +#include namespace DB { -using FunctionToLastDayOfMonth = FunctionDateOrDateTimeToSomething; +using FunctionToLastDayOfMonth = FunctionDateOrDateTimeToDateOrDate32; REGISTER_FUNCTION(ToLastDayOfMonth) { diff --git a/src/Functions/toMonday.cpp b/src/Functions/toMonday.cpp index 89145634e45..280c8a93b1a 100644 --- a/src/Functions/toMonday.cpp +++ b/src/Functions/toMonday.cpp @@ -1,12 +1,12 @@ #include #include -#include +#include namespace DB { -using FunctionToMonday = FunctionDateOrDateTimeToSomething; +using FunctionToMonday = FunctionDateOrDateTimeToDateOrDate32; REGISTER_FUNCTION(ToMonday) { diff --git a/src/Functions/toStartOfISOYear.cpp b/src/Functions/toStartOfISOYear.cpp index 366ba8dd09f..245d043f0b6 100644 --- a/src/Functions/toStartOfISOYear.cpp +++ b/src/Functions/toStartOfISOYear.cpp @@ -1,12 +1,12 @@ #include #include -#include +#include namespace DB { -using FunctionToStartOfISOYear = FunctionDateOrDateTimeToSomething; +using FunctionToStartOfISOYear = FunctionDateOrDateTimeToDateOrDate32; REGISTER_FUNCTION(ToStartOfISOYear) { diff --git a/src/Functions/toStartOfMonth.cpp b/src/Functions/toStartOfMonth.cpp index 9674462097b..00146e25d44 100644 --- a/src/Functions/toStartOfMonth.cpp +++ b/src/Functions/toStartOfMonth.cpp @@ -1,12 +1,12 @@ #include #include -#include +#include namespace DB { -using FunctionToStartOfMonth = FunctionDateOrDateTimeToSomething; +using FunctionToStartOfMonth = FunctionDateOrDateTimeToDateOrDate32; REGISTER_FUNCTION(ToStartOfMonth) { diff --git a/src/Functions/toStartOfQuarter.cpp b/src/Functions/toStartOfQuarter.cpp index c7d69743198..74966d51584 100644 --- a/src/Functions/toStartOfQuarter.cpp +++ b/src/Functions/toStartOfQuarter.cpp @@ -1,12 +1,12 @@ #include #include -#include +#include namespace DB { -using FunctionToStartOfQuarter = FunctionDateOrDateTimeToSomething; +using FunctionToStartOfQuarter = FunctionDateOrDateTimeToDateOrDate32; REGISTER_FUNCTION(ToStartOfQuarter) { diff --git a/src/Functions/toStartOfYear.cpp b/src/Functions/toStartOfYear.cpp index 13729f2f812..27019bfd69f 100644 --- a/src/Functions/toStartOfYear.cpp +++ b/src/Functions/toStartOfYear.cpp @@ -1,12 +1,12 @@ #include #include -#include +#include namespace DB { -using FunctionToStartOfYear = FunctionDateOrDateTimeToSomething; +using FunctionToStartOfYear = FunctionDateOrDateTimeToDateOrDate32; REGISTER_FUNCTION(ToStartOfYear) { diff --git a/src/Functions/transform.cpp b/src/Functions/transform.cpp index 3337e8d40a8..62be535be85 100644 --- a/src/Functions/transform.cpp +++ b/src/Functions/transform.cpp @@ -1,6 +1,7 @@ #include #include +#include #include #include #include @@ -920,8 +921,7 @@ private: ColumnString::Offset current_dst_default_offset = 0; for (size_t i = 0; i < size; ++i) { - Field key = src[i]; - const auto * it = table.find(key.reinterpret()); + const auto * it = table.find(bit_cast(src[i])); StringRef ref; if (it) @@ -1081,6 +1081,22 @@ private: mutable Cache cache; + + static UInt64 bitCastToUInt64(const Field & x) + { + switch (x.getType()) + { + case Field::Types::UInt64: return x.get(); + case Field::Types::Int64: return x.get(); + case Field::Types::Float64: return std::bit_cast(x.get()); + case Field::Types::Bool: return x.get(); + case Field::Types::Decimal32: return x.get>().getValue(); + case Field::Types::Decimal64: return x.get>().getValue(); + default: + throw Exception("Unexpected type in function 'transform'", ErrorCodes::BAD_ARGUMENTS); + } + } + /// Can be called from different threads. It works only on the first call. void initialize(const Array & from, const Array & to, const ColumnsWithTypeAndName & arguments) const { @@ -1151,20 +1167,8 @@ private: if (key.isNull()) continue; - // Field may be of Float type, but for the purpose of bitwise - // equality we can treat them as UInt64, hence the reinterpret(). - if (to[0].getType() ==Field::Types::Decimal32) - { - table[key.reinterpret()] = (*used_to)[i].reinterpret(); - } - else if (to[0].getType() ==Field::Types::Decimal64) - { - table[key.reinterpret()] = (*used_to)[i].reinterpret(); - } - else - { - table[key.reinterpret()] = (*used_to)[i].reinterpret(); - } + /// Field may be of Float type, but for the purpose of bitwise equality we can treat them as UInt64 + table[bitCastToUInt64(key)] = bitCastToUInt64((*used_to)[i]); } } else @@ -1179,7 +1183,7 @@ private: const String & str_to = to[i].get(); StringRef ref{cache.string_pool.insert(str_to.data(), str_to.size() + 1), str_to.size() + 1}; - table[key.reinterpret()] = ref; + table[bitCastToUInt64(key)] = ref; } } } @@ -1193,7 +1197,7 @@ private: { const String & str_from = from[i].get(); StringRef ref{cache.string_pool.insert(str_from.data(), str_from.size() + 1), str_from.size() + 1}; - table[ref] = (*used_to)[i].reinterpret(); + table[ref] = bitCastToUInt64((*used_to)[i]); } } else diff --git a/src/Functions/trap.cpp b/src/Functions/trap.cpp index 5bb0ed210de..16fada0f67d 100644 --- a/src/Functions/trap.cpp +++ b/src/Functions/trap.cpp @@ -14,6 +14,7 @@ #include #include #include +#include namespace DB @@ -25,6 +26,7 @@ namespace ErrorCodes extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int BAD_ARGUMENTS; extern const int CANNOT_ALLOCATE_MEMORY; + extern const int CANNOT_DLOPEN; } @@ -136,7 +138,7 @@ public: } else if (mode == "access context") { - (void)context.getCurrentQueryId(); + (void)context->getCurrentQueryId(); } else if (mode == "stack overflow") { @@ -166,6 +168,12 @@ public: maps.push_back(map); } } + else if (mode == "dlopen") + { + void * handle = dlopen("libc.so.6", RTLD_NOW); + if (!handle) + throw Exception(ErrorCodes::CANNOT_DLOPEN, "Cannot dlopen: ({})", dlerror()); // NOLINT(concurrency-mt-unsafe) // MT-Safe on Linux, see man dlerror + } else throw Exception("Unknown trap mode", ErrorCodes::BAD_ARGUMENTS); } diff --git a/src/IO/ReadBufferFromFileDescriptor.cpp b/src/IO/ReadBufferFromFileDescriptor.cpp index ffb7bff8afb..cb4b6ca5f3e 100644 --- a/src/IO/ReadBufferFromFileDescriptor.cpp +++ b/src/IO/ReadBufferFromFileDescriptor.cpp @@ -233,7 +233,7 @@ void ReadBufferFromFileDescriptor::rewind() /// Assuming file descriptor supports 'select', check that we have data to read or wait until timeout. -bool ReadBufferFromFileDescriptor::poll(size_t timeout_microseconds) +bool ReadBufferFromFileDescriptor::poll(size_t timeout_microseconds) const { fd_set fds; FD_ZERO(&fds); diff --git a/src/IO/ReadBufferFromFileDescriptor.h b/src/IO/ReadBufferFromFileDescriptor.h index 6b68b8b6dfd..6edda460bac 100644 --- a/src/IO/ReadBufferFromFileDescriptor.h +++ b/src/IO/ReadBufferFromFileDescriptor.h @@ -66,7 +66,7 @@ public: private: /// Assuming file descriptor supports 'select', check that we have data to read or wait until timeout. - bool poll(size_t timeout_microseconds); + bool poll(size_t timeout_microseconds) const; }; diff --git a/src/IO/ReadBufferFromS3.cpp b/src/IO/ReadBufferFromS3.cpp index 380365f9b95..5efe9c61a35 100644 --- a/src/IO/ReadBufferFromS3.cpp +++ b/src/IO/ReadBufferFromS3.cpp @@ -131,18 +131,18 @@ bool ReadBufferFromS3::nextImpl() ProfileEvents::increment(ProfileEvents::ReadBufferFromS3Microseconds, watch.elapsedMicroseconds()); break; } - catch (const Exception & e) + catch (Exception & e) { watch.stop(); ProfileEvents::increment(ProfileEvents::ReadBufferFromS3Microseconds, watch.elapsedMicroseconds()); ProfileEvents::increment(ProfileEvents::ReadBufferFromS3RequestsErrors, 1); - if (const auto * s3_exception = dynamic_cast(&e)) + if (auto * s3_exception = dynamic_cast(&e)) { /// It doesn't make sense to retry Access Denied or No Such Key if (!s3_exception->isRetryableError()) { - tryLogCurrentException(log, fmt::format("while reading key: {}, from bucket: {}", key, bucket)); + s3_exception->addMessage("while reading key: {}, from bucket: {}", key, bucket); throw; } } diff --git a/src/IO/S3Common.h b/src/IO/S3Common.h index da7ecf95b78..ddc23b7aa8a 100644 --- a/src/IO/S3Common.h +++ b/src/IO/S3Common.h @@ -55,7 +55,7 @@ public: bool isRetryableError() const; private: - const Aws::S3::S3Errors code; + Aws::S3::S3Errors code; }; } diff --git a/src/IO/parseDateTimeBestEffort.cpp b/src/IO/parseDateTimeBestEffort.cpp index df3f02708a9..cc0cf2576a6 100644 --- a/src/IO/parseDateTimeBestEffort.cpp +++ b/src/IO/parseDateTimeBestEffort.cpp @@ -45,12 +45,6 @@ inline size_t readAlpha(char * res, size_t max_chars, ReadBuffer & in) return num_chars; } -#if defined(__PPC__) -#if !defined(__clang__) -#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" -#endif -#endif - template inline void readDecimalNumberImpl(T & res, const char * src) { @@ -656,12 +650,6 @@ ReturnType parseDateTime64BestEffortImpl(DateTime64 & res, UInt32 scale, ReadBuf } -#if defined(__PPC__) -#if !defined(__clang__) -#pragma GCC diagnostic pop -#endif -#endif - void parseDateTimeBestEffort(time_t & res, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone) { parseDateTimeBestEffortImpl(res, in, local_time_zone, utc_time_zone, nullptr); diff --git a/src/Interpreters/ActionsVisitor.cpp b/src/Interpreters/ActionsVisitor.cpp index f10510fadae..8e135d325e6 100644 --- a/src/Interpreters/ActionsVisitor.cpp +++ b/src/Interpreters/ActionsVisitor.cpp @@ -880,20 +880,20 @@ void ActionsMatcher::visit(const ASTFunction & node, const ASTPtr & ast, Data & { case GroupByKind::GROUPING_SETS: { - data.addFunction(std::make_shared(std::make_shared(std::move(arguments_indexes), keys_info.grouping_set_keys)), { "__grouping_set" }, column_name); + data.addFunction(std::make_shared(std::make_shared(std::move(arguments_indexes), keys_info.grouping_set_keys, data.getContext()->getSettingsRef().force_grouping_standard_compatibility)), { "__grouping_set" }, column_name); break; } case GroupByKind::ROLLUP: - data.addFunction(std::make_shared(std::make_shared(std::move(arguments_indexes), aggregation_keys_number)), { "__grouping_set" }, column_name); + data.addFunction(std::make_shared(std::make_shared(std::move(arguments_indexes), aggregation_keys_number, data.getContext()->getSettingsRef().force_grouping_standard_compatibility)), { "__grouping_set" }, column_name); break; case GroupByKind::CUBE: { - data.addFunction(std::make_shared(std::make_shared(std::move(arguments_indexes), aggregation_keys_number)), { "__grouping_set" }, column_name); + data.addFunction(std::make_shared(std::make_shared(std::move(arguments_indexes), aggregation_keys_number, data.getContext()->getSettingsRef().force_grouping_standard_compatibility)), { "__grouping_set" }, column_name); break; } case GroupByKind::ORDINARY: { - data.addFunction(std::make_shared(std::make_shared(std::move(arguments_indexes))), {}, column_name); + data.addFunction(std::make_shared(std::make_shared(std::move(arguments_indexes), data.getContext()->getSettingsRef().force_grouping_standard_compatibility)), {}, column_name); break; } default: diff --git a/src/Interpreters/AsynchronousInsertQueue.cpp b/src/Interpreters/AsynchronousInsertQueue.cpp index 88b564f1dcf..cad2200c5ec 100644 --- a/src/Interpreters/AsynchronousInsertQueue.cpp +++ b/src/Interpreters/AsynchronousInsertQueue.cpp @@ -457,17 +457,20 @@ try format->addBuffer(std::move(last_buffer)); - auto chunk = Chunk(executor.getResultColumns(), total_rows); - size_t total_bytes = chunk.bytes(); + if (total_rows) + { + auto chunk = Chunk(executor.getResultColumns(), total_rows); + size_t total_bytes = chunk.bytes(); - auto source = std::make_shared(header, std::move(chunk)); - pipeline.complete(Pipe(std::move(source))); + auto source = std::make_shared(header, std::move(chunk)); + pipeline.complete(Pipe(std::move(source))); - CompletedPipelineExecutor completed_executor(pipeline); - completed_executor.execute(); + CompletedPipelineExecutor completed_executor(pipeline); + completed_executor.execute(); - LOG_INFO(log, "Flushed {} rows, {} bytes for query '{}'", - total_rows, total_bytes, queryToString(key.query)); + LOG_INFO(log, "Flushed {} rows, {} bytes for query '{}'", + total_rows, total_bytes, queryToString(key.query)); + } for (const auto & entry : data->entries) if (!entry->isFinished()) diff --git a/src/Interpreters/AsynchronousMetrics.cpp b/src/Interpreters/AsynchronousMetrics.cpp index bccfa8f5b1e..4fe69e49546 100644 --- a/src/Interpreters/AsynchronousMetrics.cpp +++ b/src/Interpreters/AsynchronousMetrics.cpp @@ -1665,7 +1665,7 @@ void AsynchronousMetrics::updateHeavyMetricsIfNeeded(TimePoint current_time, Tim LOG_IMPL(log, log_level.first, log_level.second, "Update heavy metrics. " "Update period {} sec. " - "Update heavy metrics period {} sec. " + "Update heavy metrics period {} sec. " "Heavy metrics calculation elapsed: {} sec.", update_period.count(), heavy_metric_update_period.count(), diff --git a/src/Interpreters/CatBoostModel.cpp b/src/Interpreters/CatBoostModel.cpp deleted file mode 100644 index d5803ed9e36..00000000000 --- a/src/Interpreters/CatBoostModel.cpp +++ /dev/null @@ -1,525 +0,0 @@ -#include "CatBoostModel.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - - -namespace DB -{ - -namespace ErrorCodes -{ -extern const int LOGICAL_ERROR; -extern const int BAD_ARGUMENTS; -extern const int CANNOT_LOAD_CATBOOST_MODEL; -extern const int CANNOT_APPLY_CATBOOST_MODEL; -} - -/// CatBoost wrapper interface functions. -class CatBoostWrapperAPI -{ -public: - using ModelCalcerHandle = void; - - ModelCalcerHandle * (* ModelCalcerCreate)(); // NOLINT - - void (* ModelCalcerDelete)(ModelCalcerHandle * calcer); // NOLINT - - const char * (* GetErrorString)(); // NOLINT - - bool (* LoadFullModelFromFile)(ModelCalcerHandle * calcer, const char * filename); // NOLINT - - bool (* CalcModelPredictionFlat)(ModelCalcerHandle * calcer, size_t docCount, // NOLINT - const float ** floatFeatures, size_t floatFeaturesSize, - double * result, size_t resultSize); - - bool (* CalcModelPrediction)(ModelCalcerHandle * calcer, size_t docCount, // NOLINT - const float ** floatFeatures, size_t floatFeaturesSize, - const char *** catFeatures, size_t catFeaturesSize, - double * result, size_t resultSize); - - bool (* CalcModelPredictionWithHashedCatFeatures)(ModelCalcerHandle * calcer, size_t docCount, // NOLINT - const float ** floatFeatures, size_t floatFeaturesSize, - const int ** catFeatures, size_t catFeaturesSize, - double * result, size_t resultSize); - - int (* GetStringCatFeatureHash)(const char * data, size_t size); // NOLINT - int (* GetIntegerCatFeatureHash)(uint64_t val); // NOLINT - - size_t (* GetFloatFeaturesCount)(ModelCalcerHandle* calcer); // NOLINT - size_t (* GetCatFeaturesCount)(ModelCalcerHandle* calcer); // NOLINT - size_t (* GetTreeCount)(ModelCalcerHandle* modelHandle); // NOLINT - size_t (* GetDimensionsCount)(ModelCalcerHandle* modelHandle); // NOLINT - - bool (* CheckModelMetadataHasKey)(ModelCalcerHandle* modelHandle, const char* keyPtr, size_t keySize); // NOLINT - size_t (*GetModelInfoValueSize)(ModelCalcerHandle* modelHandle, const char* keyPtr, size_t keySize); // NOLINT - const char* (*GetModelInfoValue)(ModelCalcerHandle* modelHandle, const char* keyPtr, size_t keySize); // NOLINT -}; - - -class CatBoostModelHolder -{ -private: - CatBoostWrapperAPI::ModelCalcerHandle * handle; - const CatBoostWrapperAPI * api; -public: - explicit CatBoostModelHolder(const CatBoostWrapperAPI * api_) : api(api_) { handle = api->ModelCalcerCreate(); } - ~CatBoostModelHolder() { api->ModelCalcerDelete(handle); } - - CatBoostWrapperAPI::ModelCalcerHandle * get() { return handle; } -}; - - -/// Holds CatBoost wrapper library and provides wrapper interface. -class CatBoostLibHolder -{ -public: - explicit CatBoostLibHolder(std::string lib_path_) : lib_path(std::move(lib_path_)), lib(lib_path) { initAPI(); } - - const CatBoostWrapperAPI & getAPI() const { return api; } - const std::string & getCurrentPath() const { return lib_path; } - -private: - CatBoostWrapperAPI api; - std::string lib_path; - SharedLibrary lib; - - void initAPI() - { - load(api.ModelCalcerCreate, "ModelCalcerCreate"); - load(api.ModelCalcerDelete, "ModelCalcerDelete"); - load(api.GetErrorString, "GetErrorString"); - load(api.LoadFullModelFromFile, "LoadFullModelFromFile"); - load(api.CalcModelPredictionFlat, "CalcModelPredictionFlat"); - load(api.CalcModelPrediction, "CalcModelPrediction"); - load(api.CalcModelPredictionWithHashedCatFeatures, "CalcModelPredictionWithHashedCatFeatures"); - load(api.GetStringCatFeatureHash, "GetStringCatFeatureHash"); - load(api.GetIntegerCatFeatureHash, "GetIntegerCatFeatureHash"); - load(api.GetFloatFeaturesCount, "GetFloatFeaturesCount"); - load(api.GetCatFeaturesCount, "GetCatFeaturesCount"); - tryLoad(api.CheckModelMetadataHasKey, "CheckModelMetadataHasKey"); - tryLoad(api.GetModelInfoValueSize, "GetModelInfoValueSize"); - tryLoad(api.GetModelInfoValue, "GetModelInfoValue"); - tryLoad(api.GetTreeCount, "GetTreeCount"); - tryLoad(api.GetDimensionsCount, "GetDimensionsCount"); - } - - template - void load(T& func, const std::string & name) { func = lib.get(name); } - - template - void tryLoad(T& func, const std::string & name) { func = lib.tryGet(name); } -}; - -std::shared_ptr getCatBoostWrapperHolder(const std::string & lib_path) -{ - static std::shared_ptr ptr; - static std::mutex mutex; - - std::lock_guard lock(mutex); - - if (!ptr || ptr->getCurrentPath() != lib_path) - ptr = std::make_shared(lib_path); - - return ptr; -} - -class CatBoostModelImpl -{ -public: - CatBoostModelImpl(const CatBoostWrapperAPI * api_, const std::string & model_path) : api(api_) - { - handle = std::make_unique(api); - if (!handle) - { - throw Exception(ErrorCodes::CANNOT_LOAD_CATBOOST_MODEL, - "Cannot create CatBoost model: {}", - api->GetErrorString()); - } - if (!api->LoadFullModelFromFile(handle->get(), model_path.c_str())) - { - throw Exception(ErrorCodes::CANNOT_LOAD_CATBOOST_MODEL, - "Cannot load CatBoost model: {}", - api->GetErrorString()); - } - - float_features_count = api->GetFloatFeaturesCount(handle->get()); - cat_features_count = api->GetCatFeaturesCount(handle->get()); - tree_count = 1; - if (api->GetDimensionsCount) - tree_count = api->GetDimensionsCount(handle->get()); - } - - ColumnPtr evaluate(const ColumnRawPtrs & columns) const - { - if (columns.empty()) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Got empty columns list for CatBoost model."); - - if (columns.size() != float_features_count + cat_features_count) - throw Exception(ErrorCodes::BAD_ARGUMENTS, - "Number of columns is different with number of features: columns size {} float features size {} + cat features size {}", - columns.size(), - float_features_count, - cat_features_count); - - for (size_t i = 0; i < float_features_count; ++i) - { - if (!columns[i]->isNumeric()) - { - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Column {} should be numeric to make float feature.", i); - } - } - - bool cat_features_are_strings = true; - for (size_t i = float_features_count; i < float_features_count + cat_features_count; ++i) - { - const auto * column = columns[i]; - if (column->isNumeric()) - { - cat_features_are_strings = false; - } - else if (!(typeid_cast(column) - || typeid_cast(column))) - { - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Column {} should be numeric or string.", i); - } - } - - auto result = evalImpl(columns, cat_features_are_strings); - - if (tree_count == 1) - return result; - - size_t column_size = columns.front()->size(); - auto * result_buf = result->getData().data(); - - /// Multiple trees case. Copy data to several columns. - MutableColumns mutable_columns(tree_count); - std::vector column_ptrs(tree_count); - for (size_t i = 0; i < tree_count; ++i) - { - auto col = ColumnFloat64::create(column_size); - column_ptrs[i] = col->getData().data(); - mutable_columns[i] = std::move(col); - } - - Float64 * data = result_buf; - for (size_t row = 0; row < column_size; ++row) - { - for (size_t i = 0; i < tree_count; ++i) - { - *column_ptrs[i] = *data; - ++column_ptrs[i]; - ++data; - } - } - - return ColumnTuple::create(std::move(mutable_columns)); - } - - size_t getFloatFeaturesCount() const { return float_features_count; } - size_t getCatFeaturesCount() const { return cat_features_count; } - size_t getTreeCount() const { return tree_count; } - -private: - std::unique_ptr handle; - const CatBoostWrapperAPI * api; - size_t float_features_count; - size_t cat_features_count; - size_t tree_count; - - /// Buffer should be allocated with features_count * column->size() elements. - /// Place column elements in positions buffer[0], buffer[features_count], ... , buffer[size * features_count] - template - void placeColumnAsNumber(const IColumn * column, T * buffer, size_t features_count) const - { - size_t size = column->size(); - FieldVisitorConvertToNumber visitor; - for (size_t i = 0; i < size; ++i) - { - /// TODO: Replace with column visitor. - Field field; - column->get(i, field); - *buffer = applyVisitor(visitor, field); - buffer += features_count; - } - } - - /// Buffer should be allocated with features_count * column->size() elements. - /// Place string pointers in positions buffer[0], buffer[features_count], ... , buffer[size * features_count] - static void placeStringColumn(const ColumnString & column, const char ** buffer, size_t features_count) - { - size_t size = column.size(); - for (size_t i = 0; i < size; ++i) - { - *buffer = const_cast(column.getDataAtWithTerminatingZero(i).data); - buffer += features_count; - } - } - - /// Buffer should be allocated with features_count * column->size() elements. - /// Place string pointers in positions buffer[0], buffer[features_count], ... , buffer[size * features_count] - /// Returns PODArray which holds data (because ColumnFixedString doesn't store terminating zero). - static PODArray placeFixedStringColumn( - const ColumnFixedString & column, const char ** buffer, size_t features_count) - { - size_t size = column.size(); - size_t str_size = column.getN(); - PODArray data(size * (str_size + 1)); - char * data_ptr = data.data(); - - for (size_t i = 0; i < size; ++i) - { - auto ref = column.getDataAt(i); - memcpy(data_ptr, ref.data, ref.size); - data_ptr[ref.size] = 0; - *buffer = data_ptr; - data_ptr += ref.size + 1; - buffer += features_count; - } - - return data; - } - - /// Place columns into buffer, returns column which holds placed data. Buffer should contains column->size() values. - template - ColumnPtr placeNumericColumns(const ColumnRawPtrs & columns, - size_t offset, size_t size, const T** buffer) const - { - if (size == 0) - return nullptr; - size_t column_size = columns[offset]->size(); - auto data_column = ColumnVector::create(size * column_size); - T * data = data_column->getData().data(); - for (size_t i = 0; i < size; ++i) - { - const auto * column = columns[offset + i]; - if (column->isNumeric()) - placeColumnAsNumber(column, data + i, size); - } - - for (size_t i = 0; i < column_size; ++i) - { - *buffer = data; - ++buffer; - data += size; - } - - return data_column; - } - - /// Place columns into buffer, returns data which was used for fixed string columns. - /// Buffer should contains column->size() values, each value contains size strings. - static std::vector> placeStringColumns( - const ColumnRawPtrs & columns, size_t offset, size_t size, const char ** buffer) - { - if (size == 0) - return {}; - - std::vector> data; - for (size_t i = 0; i < size; ++i) - { - const auto * column = columns[offset + i]; - if (const auto * column_string = typeid_cast(column)) - placeStringColumn(*column_string, buffer + i, size); - else if (const auto * column_fixed_string = typeid_cast(column)) - data.push_back(placeFixedStringColumn(*column_fixed_string, buffer + i, size)); - else - throw Exception("Cannot place string column.", ErrorCodes::LOGICAL_ERROR); - } - - return data; - } - - /// Calc hash for string cat feature at ps positions. - template - void calcStringHashes(const Column * column, size_t ps, const int ** buffer) const - { - size_t column_size = column->size(); - for (size_t j = 0; j < column_size; ++j) - { - auto ref = column->getDataAt(j); - const_cast(*buffer)[ps] = api->GetStringCatFeatureHash(ref.data, ref.size); - ++buffer; - } - } - - /// Calc hash for int cat feature at ps position. Buffer at positions ps should contains unhashed values. - void calcIntHashes(size_t column_size, size_t ps, const int ** buffer) const - { - for (size_t j = 0; j < column_size; ++j) - { - const_cast(*buffer)[ps] = api->GetIntegerCatFeatureHash((*buffer)[ps]); - ++buffer; - } - } - - /// buffer contains column->size() rows and size columns. - /// For int cat features calc hash inplace. - /// For string cat features calc hash from column rows. - void calcHashes(const ColumnRawPtrs & columns, size_t offset, size_t size, const int ** buffer) const - { - if (size == 0) - return; - size_t column_size = columns[offset]->size(); - - std::vector> data; - for (size_t i = 0; i < size; ++i) - { - const auto * column = columns[offset + i]; - if (const auto * column_string = typeid_cast(column)) - calcStringHashes(column_string, i, buffer); - else if (const auto * column_fixed_string = typeid_cast(column)) - calcStringHashes(column_fixed_string, i, buffer); - else - calcIntHashes(column_size, i, buffer); - } - } - - /// buffer[column_size * cat_features_count] -> char * => cat_features[column_size][cat_features_count] -> char * - void fillCatFeaturesBuffer(const char *** cat_features, const char ** buffer, - size_t column_size) const - { - for (size_t i = 0; i < column_size; ++i) - { - *cat_features = buffer; - ++cat_features; - buffer += cat_features_count; - } - } - - /// Convert values to row-oriented format and call evaluation function from CatBoost wrapper api. - /// * CalcModelPredictionFlat if no cat features - /// * CalcModelPrediction if all cat features are strings - /// * CalcModelPredictionWithHashedCatFeatures if has int cat features. - ColumnFloat64::MutablePtr evalImpl( - const ColumnRawPtrs & columns, - bool cat_features_are_strings) const - { - std::string error_msg = "Error occurred while applying CatBoost model: "; - size_t column_size = columns.front()->size(); - - auto result = ColumnFloat64::create(column_size * tree_count); - auto * result_buf = result->getData().data(); - - if (!column_size) - return result; - - /// Prepare float features. - PODArray float_features(column_size); - auto * float_features_buf = float_features.data(); - /// Store all float data into single column. float_features is a list of pointers to it. - auto float_features_col = placeNumericColumns(columns, 0, float_features_count, float_features_buf); - - if (cat_features_count == 0) - { - if (!api->CalcModelPredictionFlat(handle->get(), column_size, - float_features_buf, float_features_count, - result_buf, column_size * tree_count)) - { - - throw Exception(error_msg + api->GetErrorString(), ErrorCodes::CANNOT_APPLY_CATBOOST_MODEL); - } - return result; - } - - /// Prepare cat features. - if (cat_features_are_strings) - { - /// cat_features_holder stores pointers to ColumnString data or fixed_strings_data. - PODArray cat_features_holder(cat_features_count * column_size); - PODArray cat_features(column_size); - auto * cat_features_buf = cat_features.data(); - - fillCatFeaturesBuffer(cat_features_buf, cat_features_holder.data(), column_size); - /// Fixed strings are stored without termination zero, so have to copy data into fixed_strings_data. - auto fixed_strings_data = placeStringColumns(columns, float_features_count, - cat_features_count, cat_features_holder.data()); - - if (!api->CalcModelPrediction(handle->get(), column_size, - float_features_buf, float_features_count, - cat_features_buf, cat_features_count, - result_buf, column_size * tree_count)) - { - throw Exception(error_msg + api->GetErrorString(), ErrorCodes::CANNOT_APPLY_CATBOOST_MODEL); - } - } - else - { - PODArray cat_features(column_size); - auto * cat_features_buf = cat_features.data(); - auto cat_features_col = placeNumericColumns(columns, float_features_count, - cat_features_count, cat_features_buf); - calcHashes(columns, float_features_count, cat_features_count, cat_features_buf); - if (!api->CalcModelPredictionWithHashedCatFeatures( - handle->get(), column_size, - float_features_buf, float_features_count, - cat_features_buf, cat_features_count, - result_buf, column_size * tree_count)) - { - throw Exception(error_msg + api->GetErrorString(), ErrorCodes::CANNOT_APPLY_CATBOOST_MODEL); - } - } - - return result; - } -}; - -CatBoostModel::CatBoostModel(std::string name_, std::string model_path_, std::string lib_path_, - const ExternalLoadableLifetime & lifetime_) - : name(std::move(name_)), model_path(std::move(model_path_)), lib_path(std::move(lib_path_)), lifetime(lifetime_) -{ - api_provider = getCatBoostWrapperHolder(lib_path); - api = &api_provider->getAPI(); - model = std::make_unique(api, model_path); -} - -CatBoostModel::~CatBoostModel() = default; - -size_t CatBoostModel::getFloatFeaturesCount() const -{ - return model->getFloatFeaturesCount(); -} - -size_t CatBoostModel::getCatFeaturesCount() const -{ - return model->getCatFeaturesCount(); -} - -size_t CatBoostModel::getTreeCount() const -{ - return model->getTreeCount(); -} - -DataTypePtr CatBoostModel::getReturnType() const -{ - size_t tree_count = getTreeCount(); - auto type = std::make_shared(); - if (tree_count == 1) - return type; - - DataTypes types(tree_count, type); - - return std::make_shared(types); -} - -ColumnPtr CatBoostModel::evaluate(const ColumnRawPtrs & columns) const -{ - if (!model) - throw Exception("CatBoost model was not loaded.", ErrorCodes::LOGICAL_ERROR); - - return model->evaluate(columns); -} - -} diff --git a/src/Interpreters/CatBoostModel.h b/src/Interpreters/CatBoostModel.h deleted file mode 100644 index 7bb1df92b67..00000000000 --- a/src/Interpreters/CatBoostModel.h +++ /dev/null @@ -1,73 +0,0 @@ -#pragma once - -#include -#include -#include - - -namespace DB -{ - -class CatBoostLibHolder; -class CatBoostWrapperAPI; -class CatBoostModelImpl; - -class IDataType; -using DataTypePtr = std::shared_ptr; - -/// General ML model evaluator interface. -class IMLModel : public IExternalLoadable -{ -public: - IMLModel() = default; - virtual ColumnPtr evaluate(const ColumnRawPtrs & columns) const = 0; - virtual std::string getTypeName() const = 0; - virtual DataTypePtr getReturnType() const = 0; - virtual ~IMLModel() override = default; -}; - -class CatBoostModel : public IMLModel -{ -public: - CatBoostModel(std::string name, std::string model_path, - std::string lib_path, const ExternalLoadableLifetime & lifetime); - - ~CatBoostModel() override; - - ColumnPtr evaluate(const ColumnRawPtrs & columns) const override; - std::string getTypeName() const override { return "catboost"; } - - size_t getFloatFeaturesCount() const; - size_t getCatFeaturesCount() const; - size_t getTreeCount() const; - DataTypePtr getReturnType() const override; - - /// IExternalLoadable interface. - - const ExternalLoadableLifetime & getLifetime() const override { return lifetime; } - - std::string getLoadableName() const override { return name; } - - bool supportUpdates() const override { return true; } - - bool isModified() const override { return true; } - - std::shared_ptr clone() const override - { - return std::make_shared(name, model_path, lib_path, lifetime); - } - -private: - const std::string name; - std::string model_path; - std::string lib_path; - ExternalLoadableLifetime lifetime; - std::shared_ptr api_provider; - const CatBoostWrapperAPI * api; - - std::unique_ptr model; - - void init(); -}; - -} diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index b3bd9d97005..d69878e6af0 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -52,7 +52,6 @@ #include #include #include -#include #include #include #include @@ -153,7 +152,6 @@ struct ContextSharedPart : boost::noncopyable mutable std::mutex embedded_dictionaries_mutex; mutable std::mutex external_dictionaries_mutex; mutable std::mutex external_user_defined_executable_functions_mutex; - mutable std::mutex external_models_mutex; /// Separate mutex for storage policies. During server startup we may /// initialize some important storages (system logs with MergeTree engine) /// under context lock. @@ -191,9 +189,7 @@ struct ContextSharedPart : boost::noncopyable mutable std::unique_ptr embedded_dictionaries; /// Metrica's dictionaries. Have lazy initialization. mutable std::unique_ptr external_dictionaries_loader; mutable std::unique_ptr external_user_defined_executable_functions_loader; - mutable std::unique_ptr external_models_loader; - ExternalLoaderXMLConfigRepository * external_models_config_repository = nullptr; scope_guard models_repository_guard; ExternalLoaderXMLConfigRepository * external_dictionaries_config_repository = nullptr; @@ -359,8 +355,6 @@ struct ContextSharedPart : boost::noncopyable external_dictionaries_loader->enablePeriodicUpdates(false); if (external_user_defined_executable_functions_loader) external_user_defined_executable_functions_loader->enablePeriodicUpdates(false); - if (external_models_loader) - external_models_loader->enablePeriodicUpdates(false); Session::shutdownNamedSessions(); @@ -391,7 +385,6 @@ struct ContextSharedPart : boost::noncopyable std::unique_ptr delete_embedded_dictionaries; std::unique_ptr delete_external_dictionaries_loader; std::unique_ptr delete_external_user_defined_executable_functions_loader; - std::unique_ptr delete_external_models_loader; std::unique_ptr delete_buffer_flush_schedule_pool; std::unique_ptr delete_schedule_pool; std::unique_ptr delete_distributed_schedule_pool; @@ -430,7 +423,6 @@ struct ContextSharedPart : boost::noncopyable delete_embedded_dictionaries = std::move(embedded_dictionaries); delete_external_dictionaries_loader = std::move(external_dictionaries_loader); delete_external_user_defined_executable_functions_loader = std::move(external_user_defined_executable_functions_loader); - delete_external_models_loader = std::move(external_models_loader); delete_buffer_flush_schedule_pool = std::move(buffer_flush_schedule_pool); delete_schedule_pool = std::move(schedule_pool); delete_distributed_schedule_pool = std::move(distributed_schedule_pool); @@ -458,7 +450,6 @@ struct ContextSharedPart : boost::noncopyable delete_embedded_dictionaries.reset(); delete_external_dictionaries_loader.reset(); delete_external_user_defined_executable_functions_loader.reset(); - delete_external_models_loader.reset(); delete_ddl_worker.reset(); delete_buffer_flush_schedule_pool.reset(); delete_schedule_pool.reset(); @@ -1476,48 +1467,6 @@ ExternalUserDefinedExecutableFunctionsLoader & Context::getExternalUserDefinedEx return *shared->external_user_defined_executable_functions_loader; } -const ExternalModelsLoader & Context::getExternalModelsLoader() const -{ - return const_cast(this)->getExternalModelsLoader(); -} - -ExternalModelsLoader & Context::getExternalModelsLoader() -{ - std::lock_guard lock(shared->external_models_mutex); - return getExternalModelsLoaderUnlocked(); -} - -ExternalModelsLoader & Context::getExternalModelsLoaderUnlocked() -{ - if (!shared->external_models_loader) - shared->external_models_loader = - std::make_unique(getGlobalContext()); - return *shared->external_models_loader; -} - -void Context::loadOrReloadModels(const Poco::Util::AbstractConfiguration & config) -{ - auto patterns_values = getMultipleValuesFromConfig(config, "", "models_config"); - std::unordered_set patterns(patterns_values.begin(), patterns_values.end()); - - std::lock_guard lock(shared->external_models_mutex); - - auto & external_models_loader = getExternalModelsLoaderUnlocked(); - - if (shared->external_models_config_repository) - { - shared->external_models_config_repository->updatePatterns(patterns); - external_models_loader.reloadConfig(shared->external_models_config_repository->getName()); - return; - } - - auto app_path = getPath(); - auto config_path = getConfigRef().getString("config-file", "config.xml"); - auto repository = std::make_unique(app_path, config_path, patterns); - shared->external_models_config_repository = repository.get(); - shared->models_repository_guard = external_models_loader.addConfigRepository(std::move(repository)); -} - EmbeddedDictionaries & Context::getEmbeddedDictionariesImpl(const bool throw_on_error) const { std::lock_guard lock(shared->embedded_dictionaries_mutex); diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index db3f701bf74..67cf584d5a7 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -53,7 +53,6 @@ class AccessRightsElements; enum class RowPolicyFilterType; class EmbeddedDictionaries; class ExternalDictionariesLoader; -class ExternalModelsLoader; class ExternalUserDefinedExecutableFunctionsLoader; class InterserverCredentials; using InterserverCredentialsPtr = std::shared_ptr; @@ -612,6 +611,7 @@ public: void killCurrentQuery(); + bool hasInsertionTable() const { return !insertion_table.empty(); } void setInsertionTable(StorageID db_and_table) { insertion_table = std::move(db_and_table); } const StorageID & getInsertionTable() const { return insertion_table; } @@ -644,19 +644,15 @@ public: const EmbeddedDictionaries & getEmbeddedDictionaries() const; const ExternalDictionariesLoader & getExternalDictionariesLoader() const; - const ExternalModelsLoader & getExternalModelsLoader() const; const ExternalUserDefinedExecutableFunctionsLoader & getExternalUserDefinedExecutableFunctionsLoader() const; EmbeddedDictionaries & getEmbeddedDictionaries(); ExternalDictionariesLoader & getExternalDictionariesLoader(); ExternalDictionariesLoader & getExternalDictionariesLoaderUnlocked(); ExternalUserDefinedExecutableFunctionsLoader & getExternalUserDefinedExecutableFunctionsLoader(); ExternalUserDefinedExecutableFunctionsLoader & getExternalUserDefinedExecutableFunctionsLoaderUnlocked(); - ExternalModelsLoader & getExternalModelsLoader(); - ExternalModelsLoader & getExternalModelsLoaderUnlocked(); void tryCreateEmbeddedDictionaries(const Poco::Util::AbstractConfiguration & config) const; void loadOrReloadDictionaries(const Poco::Util::AbstractConfiguration & config); void loadOrReloadUserDefinedExecutableFunctions(const Poco::Util::AbstractConfiguration & config); - void loadOrReloadModels(const Poco::Util::AbstractConfiguration & config); #if USE_NLP SynonymsExtensions & getSynonymsExtensions() const; diff --git a/src/Interpreters/DDLWorker.cpp b/src/Interpreters/DDLWorker.cpp index 6ec20ab5f5f..c8878297c02 100644 --- a/src/Interpreters/DDLWorker.cpp +++ b/src/Interpreters/DDLWorker.cpp @@ -890,7 +890,7 @@ void DDLWorker::cleanupQueue(Int64, const ZooKeeperPtr & zookeeper) /// We recursively delete all nodes except node_path/finished to prevent staled hosts from /// creating node_path/active node (see createStatusDirs(...)) - zookeeper->tryRemoveChildrenRecursive(node_path, /* probably_flat */ false, "finished"); + zookeeper->tryRemoveChildrenRecursive(node_path, /* probably_flat */ false, zkutil::RemoveException{"finished"}); /// And then we remove node_path and node_path/finished in a single transaction Coordination::Requests ops; diff --git a/src/Interpreters/DDLWorker.h b/src/Interpreters/DDLWorker.h index 7ddcc80c02a..e3c1fa4c271 100644 --- a/src/Interpreters/DDLWorker.h +++ b/src/Interpreters/DDLWorker.h @@ -61,18 +61,23 @@ public: return host_fqdn_id; } + std::string getQueueDir() const + { + return queue_dir; + } + void startup(); virtual void shutdown(); bool isCurrentlyActive() const { return initialized && !stop_flag; } -protected: /// Returns cached ZooKeeper session (possibly expired). ZooKeeperPtr tryGetZooKeeper() const; /// If necessary, creates a new session and caches it. ZooKeeperPtr getAndSetZooKeeper(); +protected: /// Iterates through queue tasks in ZooKeeper, runs execution of new tasks void scheduleTasks(bool reinitialized); diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index be32125edf8..9daa42bf499 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -1988,6 +1988,23 @@ ExpressionAnalysisResult::ExpressionAnalysisResult( } } + // Here we need to set order by expression as required output to avoid + // their removal from the ActionsDAG. + const auto * select_query = query_analyzer.getSelectQuery(); + if (select_query->orderBy()) + { + for (auto & child : select_query->orderBy()->children) + { + auto * ast = child->as(); + ASTPtr order_expression = ast->children.at(0); + if (auto * function = order_expression->as(); + function && (function->is_window_function || function->compute_after_window_functions)) + continue; + const String & column_name = order_expression->getColumnName(); + chain.getLastStep().addRequiredOutput(column_name); + } + } + before_window = chain.getLastActions(); finalize_chain(chain); @@ -2007,7 +2024,6 @@ ExpressionAnalysisResult::ExpressionAnalysisResult( // produced the expressions required to calculate window functions. // They are not needed in the final SELECT result. Knowing the correct // list of columns is important when we apply SELECT DISTINCT later. - const auto * select_query = query_analyzer.getSelectQuery(); for (const auto & child : select_query->select()->children) { step.addRequiredOutput(child->getColumnName()); diff --git a/src/Interpreters/ExternalModelsLoader.cpp b/src/Interpreters/ExternalModelsLoader.cpp deleted file mode 100644 index 317cf0bf1c9..00000000000 --- a/src/Interpreters/ExternalModelsLoader.cpp +++ /dev/null @@ -1,41 +0,0 @@ -#include -#include - -namespace DB -{ - -namespace ErrorCodes -{ - extern const int INVALID_CONFIG_PARAMETER; -} - - -ExternalModelsLoader::ExternalModelsLoader(ContextPtr context_) - : ExternalLoader("external model", &Poco::Logger::get("ExternalModelsLoader")), WithContext(context_) -{ - setConfigSettings({"model", "name", {}, {}}); - enablePeriodicUpdates(true); -} - -std::shared_ptr ExternalModelsLoader::create( - const std::string & name, const Poco::Util::AbstractConfiguration & config, - const std::string & config_prefix, const std::string & /* repository_name */) const -{ - String type = config.getString(config_prefix + ".type"); - ExternalLoadableLifetime lifetime(config, config_prefix + ".lifetime"); - - /// TODO: add models factory. - if (type == "catboost") - { - return std::make_unique( - name, config.getString(config_prefix + ".path"), - getContext()->getConfigRef().getString("catboost_dynamic_library_path"), - lifetime - ); - } - else - { - throw Exception("Unknown model type: " + type, ErrorCodes::INVALID_CONFIG_PARAMETER); - } -} -} diff --git a/src/Interpreters/ExternalModelsLoader.h b/src/Interpreters/ExternalModelsLoader.h deleted file mode 100644 index 0eeb60008c3..00000000000 --- a/src/Interpreters/ExternalModelsLoader.h +++ /dev/null @@ -1,40 +0,0 @@ -#pragma once - -#include -#include -#include -#include - -#include - - -namespace DB -{ - -/// Manages user-defined models. -class ExternalModelsLoader : public ExternalLoader, WithContext -{ -public: - using ModelPtr = std::shared_ptr; - - /// Models will be loaded immediately and then will be updated in separate thread, each 'reload_period' seconds. - explicit ExternalModelsLoader(ContextPtr context_); - - ModelPtr getModel(const std::string & model_name) const - { - return std::static_pointer_cast(load(model_name)); - } - - void reloadModel(const std::string & model_name) const - { - loadOrReload(model_name); - } - -protected: - LoadablePtr create(const std::string & name, const Poco::Util::AbstractConfiguration & config, - const std::string & config_prefix, const std::string & repository_name) const override; - - friend class StorageSystemModels; -}; - -} diff --git a/src/Interpreters/InterpreterSystemQuery.cpp b/src/Interpreters/InterpreterSystemQuery.cpp index 18cd159ae0e..34504933436 100644 --- a/src/Interpreters/InterpreterSystemQuery.cpp +++ b/src/Interpreters/InterpreterSystemQuery.cpp @@ -12,7 +12,6 @@ #include #include #include -#include #include #include #include @@ -36,6 +35,7 @@ #include #include #include +#include #include #include #include @@ -387,17 +387,15 @@ BlockIO InterpreterSystemQuery::execute() case Type::RELOAD_MODEL: { getContext()->checkAccess(AccessType::SYSTEM_RELOAD_MODEL); - - auto & external_models_loader = system_context->getExternalModelsLoader(); - external_models_loader.reloadModel(query.target_model); + auto bridge_helper = std::make_unique(getContext(), query.target_model); + bridge_helper->removeModel(); break; } case Type::RELOAD_MODELS: { getContext()->checkAccess(AccessType::SYSTEM_RELOAD_MODEL); - - auto & external_models_loader = system_context->getExternalModelsLoader(); - external_models_loader.reloadAllTriedToLoad(); + auto bridge_helper = std::make_unique(getContext()); + bridge_helper->removeAllModels(); break; } case Type::RELOAD_FUNCTION: diff --git a/src/Interpreters/JIT/compileFunction.cpp b/src/Interpreters/JIT/compileFunction.cpp index 353ab84674c..99646084e5a 100644 --- a/src/Interpreters/JIT/compileFunction.cpp +++ b/src/Interpreters/JIT/compileFunction.cpp @@ -739,7 +739,10 @@ CompiledAggregateFunctions compileAggregateFunctions(CHJIT & jit, const std::vec { compileCreateAggregateStatesFunctions(module, functions, create_aggregate_states_functions_name); compileAddIntoAggregateStatesFunctions(module, functions, add_aggregate_states_functions_name); - compileAddIntoAggregateStatesFunctionsSinglePlace(module, functions, add_aggregate_states_functions_name_single_place); + /// FIXME: this leads to use-of-uninitialized-value in llvm + /// But for now, it is safe, since it is not used by Aggregator anyway + (void)compileAddIntoAggregateStatesFunctionsSinglePlace; + /// compileAddIntoAggregateStatesFunctionsSinglePlace(module, functions, add_aggregate_states_functions_name_single_place); compileMergeAggregatesStates(module, functions, merge_aggregate_states_functions_name); compileInsertAggregatesIntoResultColumns(module, functions, insert_aggregate_states_functions_name); }); @@ -752,7 +755,7 @@ CompiledAggregateFunctions compileAggregateFunctions(CHJIT & jit, const std::vec assert(create_aggregate_states_function); assert(add_into_aggregate_states_function); - assert(add_into_aggregate_states_function_single_place); + /// assert(add_into_aggregate_states_function_single_place); /// FIXME assert(merge_aggregate_states_function); assert(insert_aggregate_states_function); diff --git a/src/Interpreters/MutationsInterpreter.cpp b/src/Interpreters/MutationsInterpreter.cpp index c496995ba65..26b8bce1f4a 100644 --- a/src/Interpreters/MutationsInterpreter.cpp +++ b/src/Interpreters/MutationsInterpreter.cpp @@ -473,6 +473,7 @@ ASTPtr MutationsInterpreter::prepare(bool dry_run) dependencies = getAllColumnDependencies(metadata_snapshot, updated_columns); + std::vector read_columns; /// First, break a sequence of commands into stages. for (auto & command : commands) { @@ -706,17 +707,23 @@ ASTPtr MutationsInterpreter::prepare(bool dry_run) else if (command.type == MutationCommand::READ_COLUMN) { mutation_kind.set(MutationKind::MUTATE_OTHER); - if (stages.empty() || !stages.back().column_to_updated.empty()) - stages.emplace_back(context); - if (stages.size() == 1) /// First stage only supports filtering and can't update columns. - stages.emplace_back(context); - - stages.back().column_to_updated.emplace(command.column_name, std::make_shared(command.column_name)); + read_columns.emplace_back(command.column_name); } else throw Exception("Unknown mutation command type: " + DB::toString(command.type), ErrorCodes::UNKNOWN_MUTATION_COMMAND); } + if (!read_columns.empty()) + { + if (stages.empty() || !stages.back().column_to_updated.empty()) + stages.emplace_back(context); + if (stages.size() == 1) /// First stage only supports filtering and can't update columns. + stages.emplace_back(context); + + for (auto & column_name : read_columns) + stages.back().column_to_updated.emplace(column_name, std::make_shared(column_name)); + } + /// We care about affected indices and projections because we also need to rewrite them /// when one of index columns updated or filtered with delete. /// The same about columns, that are needed for calculation of TTL expressions. diff --git a/src/Interpreters/SortedBlocksWriter.cpp b/src/Interpreters/SortedBlocksWriter.cpp index 82d501451a6..8f598f3dd3f 100644 --- a/src/Interpreters/SortedBlocksWriter.cpp +++ b/src/Interpreters/SortedBlocksWriter.cpp @@ -28,6 +28,11 @@ namespace CurrentMetrics namespace DB { +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + namespace { @@ -84,10 +89,13 @@ void SortedBlocksWriter::insert(Block && block) size_t bytes = 0; size_t flush_no = 0; + if (!block.rows()) + return; + { std::lock_guard lock{insert_mutex}; - /// insert bock into BlocksList undef lock + /// insert block into BlocksList under lock inserted_blocks.insert(std::move(block)); size_t total_row_count = inserted_blocks.row_count + row_count_in_flush; @@ -145,7 +153,7 @@ SortedBlocksWriter::TmpFilePtr SortedBlocksWriter::flush(const BlocksList & bloc pipes.emplace_back(std::make_shared(block.cloneEmpty(), Chunk(block.getColumns(), num_rows))); if (pipes.empty()) - return {}; + throw Exception(ErrorCodes::LOGICAL_ERROR, "Empty block"); QueryPipelineBuilder pipeline; pipeline.init(Pipe::unitePipes(std::move(pipes))); diff --git a/src/Interpreters/StorageID.h b/src/Interpreters/StorageID.h index c60c3aec9c6..43710988243 100644 --- a/src/Interpreters/StorageID.h +++ b/src/Interpreters/StorageID.h @@ -69,6 +69,8 @@ struct StorageID return uuid != UUIDHelpers::Nil; } + bool hasDatabase() const { return !database_name.empty(); } + bool operator<(const StorageID & rhs) const; bool operator==(const StorageID & rhs) const; diff --git a/src/Interpreters/TreeOptimizer.cpp b/src/Interpreters/TreeOptimizer.cpp index 3f7e141db3e..74f084df40b 100644 --- a/src/Interpreters/TreeOptimizer.cpp +++ b/src/Interpreters/TreeOptimizer.cpp @@ -418,7 +418,7 @@ void optimizeDuplicateDistinct(ASTSelectQuery & select) return; std::unordered_set distinct_names = getDistinctNames(*subselect); - std::unordered_set selected_names; + std::unordered_set selected_names; /// Check source column names from select list (ignore aliases and table names) for (const auto & id : select.select()->children) @@ -427,11 +427,11 @@ void optimizeDuplicateDistinct(ASTSelectQuery & select) if (!identifier) return; - String name = identifier->shortName(); + const String & name = identifier->shortName(); if (!distinct_names.contains(name)) return; /// Not a distinct column, keep DISTINCT for it. - selected_names.insert(name); + selected_names.emplace(name); } /// select columns list != distinct columns list diff --git a/src/Interpreters/UserDefinedExecutableFunctionFactory.cpp b/src/Interpreters/UserDefinedExecutableFunctionFactory.cpp index 8e4b66ef893..18784609397 100644 --- a/src/Interpreters/UserDefinedExecutableFunctionFactory.cpp +++ b/src/Interpreters/UserDefinedExecutableFunctionFactory.cpp @@ -28,16 +28,18 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; } +namespace +{ + class UserDefinedFunction final : public IFunction { public: - explicit UserDefinedFunction( ExternalUserDefinedExecutableFunctionsLoader::UserDefinedExecutableFunctionPtr executable_function_, ContextPtr context_, Array parameters_) : executable_function(std::move(executable_function_)) - , context(context_) + , context(std::move(context_)) { const auto & configuration = executable_function->getConfiguration(); size_t command_parameters_size = configuration.parameters.size(); @@ -230,6 +232,8 @@ private: std::vector command_arguments_with_parameters; }; +} + UserDefinedExecutableFunctionFactory & UserDefinedExecutableFunctionFactory::instance() { static UserDefinedExecutableFunctionFactory result; diff --git a/src/Interpreters/convertFieldToType.cpp b/src/Interpreters/convertFieldToType.cpp index dd23ad69ae2..4e7562ef451 100644 --- a/src/Interpreters/convertFieldToType.cpp +++ b/src/Interpreters/convertFieldToType.cpp @@ -6,26 +6,24 @@ #include #include #include -#include -#include #include #include #include -#include #include #include #include #include #include #include +#include #include + #include #include #include - +#include #include -#include namespace DB @@ -223,7 +221,7 @@ Field convertFieldToTypeImpl(const Field & src, const IDataType & type, const ID && (which_from_type.isNativeInt() || which_from_type.isNativeUInt() || which_from_type.isDate() || which_from_type.isDate32() || which_from_type.isDateTime() || which_from_type.isDateTime64())) { const auto scale = static_cast(type).getScale(); - const auto decimal_value = DecimalUtils::decimalFromComponents(src.reinterpret(), 0, scale); + const auto decimal_value = DecimalUtils::decimalFromComponents(applyVisitor(FieldVisitorConvertToNumber(), src), 0, scale); return Field(DecimalField(decimal_value, scale)); } } diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index c501c1722ba..b6434955418 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -838,101 +838,117 @@ static std::tuple executeQueryImpl( { QueryStatus * process_list_elem = context->getProcessListElement(); - if (!process_list_elem) - return; - - /// Update performance counters before logging to query_log - CurrentThread::finalizePerformanceCounters(); - - QueryStatusInfo info = process_list_elem->getInfo(true, context->getSettingsRef().log_profile_events); - - double elapsed_seconds = info.elapsed_seconds; - - elem.type = QueryLogElementType::QUERY_FINISH; - - // construct event_time and event_time_microseconds using the same time point - // so that the two times will always be equal up to a precision of a second. - const auto finish_time = std::chrono::system_clock::now(); - elem.event_time = time_in_seconds(finish_time); - elem.event_time_microseconds = time_in_microseconds(finish_time); - status_info_to_query_log(elem, info, ast, context); - - if (pulling_pipeline) + if (process_list_elem) { - query_pipeline.tryGetResultRowsAndBytes(elem.result_rows, elem.result_bytes); - } - else /// will be used only for ordinary INSERT queries - { - auto progress_out = process_list_elem->getProgressOut(); - elem.result_rows = progress_out.written_rows; - elem.result_bytes = progress_out.written_bytes; - } + /// Update performance counters before logging to query_log + CurrentThread::finalizePerformanceCounters(); - auto progress_callback = context->getProgressCallback(); - if (progress_callback) - { - Progress p(WriteProgress{info.written_rows, info.written_bytes}); - p.incrementPiecewiseAtomically(Progress{ResultProgress{elem.result_rows, elem.result_bytes}}); - progress_callback(p); - } + QueryStatusInfo info = process_list_elem->getInfo(true, context->getSettingsRef().log_profile_events); - if (elem.read_rows != 0) - { - LOG_INFO(&Poco::Logger::get("executeQuery"), "Read {} rows, {} in {} sec., {} rows/sec., {}/sec.", - elem.read_rows, ReadableSize(elem.read_bytes), elapsed_seconds, - static_cast(elem.read_rows / elapsed_seconds), - ReadableSize(elem.read_bytes / elapsed_seconds)); - } + double elapsed_seconds = info.elapsed_seconds; - if (log_queries && elem.type >= log_queries_min_type && static_cast(elem.query_duration_ms) >= log_queries_min_query_duration_ms) - { - if (auto query_log = context->getQueryLog()) - query_log->add(elem); - } - if (log_processors_profiles) - { - if (auto processors_profile_log = context->getProcessorsProfileLog()) + elem.type = QueryLogElementType::QUERY_FINISH; + + // construct event_time and event_time_microseconds using the same time point + // so that the two times will always be equal up to a precision of a second. + const auto finish_time = std::chrono::system_clock::now(); + elem.event_time = time_in_seconds(finish_time); + elem.event_time_microseconds = time_in_microseconds(finish_time); + status_info_to_query_log(elem, info, ast, context); + + if (pulling_pipeline) { - ProcessorProfileLogElement processor_elem; - processor_elem.event_time = time_in_seconds(finish_time); - processor_elem.event_time_microseconds = time_in_microseconds(finish_time); - processor_elem.query_id = elem.client_info.current_query_id; + query_pipeline.tryGetResultRowsAndBytes(elem.result_rows, elem.result_bytes); + } + else /// will be used only for ordinary INSERT queries + { + auto progress_out = process_list_elem->getProgressOut(); + elem.result_rows = progress_out.written_rows; + elem.result_bytes = progress_out.written_bytes; + } - auto get_proc_id = [](const IProcessor & proc) -> UInt64 - { - return reinterpret_cast(&proc); - }; + auto progress_callback = context->getProgressCallback(); + if (progress_callback) + { + Progress p(WriteProgress{info.written_rows, info.written_bytes}); + p.incrementPiecewiseAtomically(Progress{ResultProgress{elem.result_rows, elem.result_bytes}}); + progress_callback(p); + } - for (const auto & processor : query_pipeline.getProcessors()) + if (elem.read_rows != 0) + { + LOG_INFO(&Poco::Logger::get("executeQuery"), "Read {} rows, {} in {} sec., {} rows/sec., {}/sec.", + elem.read_rows, ReadableSize(elem.read_bytes), elapsed_seconds, + static_cast(elem.read_rows / elapsed_seconds), + ReadableSize(elem.read_bytes / elapsed_seconds)); + } + + if (log_queries && elem.type >= log_queries_min_type && static_cast(elem.query_duration_ms) >= log_queries_min_query_duration_ms) + { + if (auto query_log = context->getQueryLog()) + query_log->add(elem); + } + if (log_processors_profiles) + { + if (auto processors_profile_log = context->getProcessorsProfileLog()) { - std::vector parents; - for (const auto & port : processor->getOutputs()) + ProcessorProfileLogElement processor_elem; + processor_elem.event_time = time_in_seconds(finish_time); + processor_elem.event_time_microseconds = time_in_microseconds(finish_time); + processor_elem.query_id = elem.client_info.current_query_id; + + auto get_proc_id = [](const IProcessor & proc) -> UInt64 { - if (!port.isConnected()) - continue; - const IProcessor & next = port.getInputPort().getProcessor(); - parents.push_back(get_proc_id(next)); + return reinterpret_cast(&proc); + }; + + for (const auto & processor : query_pipeline.getProcessors()) + { + std::vector parents; + for (const auto & port : processor->getOutputs()) + { + if (!port.isConnected()) + continue; + const IProcessor & next = port.getInputPort().getProcessor(); + parents.push_back(get_proc_id(next)); + } + + processor_elem.id = get_proc_id(*processor); + processor_elem.parent_ids = std::move(parents); + + processor_elem.plan_step = reinterpret_cast(processor->getQueryPlanStep()); + processor_elem.plan_group = processor->getQueryPlanStepGroup(); + + processor_elem.processor_name = processor->getName(); + + processor_elem.elapsed_us = processor->getElapsedUs(); + processor_elem.input_wait_elapsed_us = processor->getInputWaitElapsedUs(); + processor_elem.output_wait_elapsed_us = processor->getOutputWaitElapsedUs(); + + auto stats = processor->getProcessorDataStats(); + processor_elem.input_rows = stats.input_rows; + processor_elem.input_bytes = stats.input_bytes; + processor_elem.output_rows = stats.output_rows; + processor_elem.output_bytes = stats.output_bytes; + + processors_profile_log->add(processor_elem); } + } + } - processor_elem.id = get_proc_id(*processor); - processor_elem.parent_ids = std::move(parents); - - processor_elem.plan_step = reinterpret_cast(processor->getQueryPlanStep()); - processor_elem.plan_group = processor->getQueryPlanStepGroup(); - - processor_elem.processor_name = processor->getName(); - - processor_elem.elapsed_us = processor->getElapsedUs(); - processor_elem.input_wait_elapsed_us = processor->getInputWaitElapsedUs(); - processor_elem.output_wait_elapsed_us = processor->getOutputWaitElapsedUs(); - - auto stats = processor->getProcessorDataStats(); - processor_elem.input_rows = stats.input_rows; - processor_elem.input_bytes = stats.input_bytes; - processor_elem.output_rows = stats.output_rows; - processor_elem.output_bytes = stats.output_bytes; - - processors_profile_log->add(processor_elem); + if (implicit_txn_control) + { + try + { + implicit_txn_control->executeCommit(context->getSessionContext()); + implicit_txn_control.reset(); + } + catch (const Exception &) + { + /// An exception might happen when trying to commit the transaction. For example we might get an immediate exception + /// because ZK is down and wait_changes_become_visible_after_commit_mode == WAIT_UNKNOWN + implicit_txn_control.reset(); + throw; } } } @@ -945,27 +961,11 @@ static std::tuple executeQueryImpl( query_span->addAttributeIfNotEmpty("clickhouse.tracestate", OpenTelemetry::CurrentContext().tracestate); query_span->addAttributeIfNotZero("clickhouse.read_rows", elem.read_rows); query_span->addAttributeIfNotZero("clickhouse.read_bytes", elem.read_bytes); - query_span->addAttributeIfNotZero("clickhouse.written_rows", info.written_rows); + query_span->addAttributeIfNotZero("clickhouse.written_rows", elem.written_rows); query_span->addAttributeIfNotZero("clickhouse.written_bytes", elem.written_bytes); query_span->addAttributeIfNotZero("clickhouse.memory_usage", elem.memory_usage); query_span->finish(); } - - if (implicit_txn_control) - { - try - { - implicit_txn_control->executeCommit(context->getSessionContext()); - implicit_txn_control.reset(); - } - catch (const Exception &) - { - /// An exception might happen when trying to commit the transaction. For example we might get an immediate exception - /// because ZK is down and wait_changes_become_visible_after_commit_mode == WAIT_UNKNOWN - implicit_txn_control.reset(); - throw; - } - } }; auto exception_callback = [elem, diff --git a/src/Loggers/CMakeLists.txt b/src/Loggers/CMakeLists.txt index 02a886765a1..1f32d747b01 100644 --- a/src/Loggers/CMakeLists.txt +++ b/src/Loggers/CMakeLists.txt @@ -2,9 +2,9 @@ add_headers_and_sources(loggers .) # Standard version depends on DBMS and works with text log add_library(loggers ${loggers_sources} ${loggers_headers}) -target_compile_definitions(loggers PUBLIC WITH_TEXT_LOG=1) target_link_libraries(loggers PRIVATE dbms clickhouse_common_io) # Lightweight version doesn't work with textlog and also doesn't depend on DBMS add_library(loggers_no_text_log ${loggers_sources} ${loggers_headers}) +target_compile_definitions(loggers_no_text_log PUBLIC WITHOUT_TEXT_LOG=1) target_link_libraries(loggers_no_text_log PRIVATE clickhouse_common_io) diff --git a/src/Loggers/Loggers.cpp b/src/Loggers/Loggers.cpp index 97e26f2c038..b3bd8d588cd 100644 --- a/src/Loggers/Loggers.cpp +++ b/src/Loggers/Loggers.cpp @@ -10,7 +10,7 @@ #include #include -#ifdef WITH_TEXT_LOG +#ifndef WITHOUT_TEXT_LOG #include #endif @@ -34,7 +34,7 @@ static std::string createDirectory(const std::string & file) return path; } -#ifdef WITH_TEXT_LOG +#ifndef WITHOUT_TEXT_LOG void Loggers::setTextLog(std::shared_ptr log, int max_priority) { text_log = log; @@ -44,7 +44,7 @@ void Loggers::setTextLog(std::shared_ptr log, int max_priority) void Loggers::buildLoggers(Poco::Util::AbstractConfiguration & config, Poco::Logger & logger /*_root*/, const std::string & cmd_name) { -#ifdef WITH_TEXT_LOG +#ifndef WITHOUT_TEXT_LOG if (split) if (auto log = text_log.lock()) split->addTextLog(log, text_log_max_priority); diff --git a/src/Loggers/Loggers.h b/src/Loggers/Loggers.h index 22b2b5e2c69..31a215aa9ce 100644 --- a/src/Loggers/Loggers.h +++ b/src/Loggers/Loggers.h @@ -7,7 +7,7 @@ #include #include "OwnSplitChannel.h" -#ifdef WITH_TEXT_LOG +#ifndef WITHOUT_TEXT_LOG namespace DB { class TextLog; @@ -29,7 +29,7 @@ public: /// Close log files. On next log write files will be reopened. void closeLogs(Poco::Logger & logger); -#ifdef WITH_TEXT_LOG +#ifndef WITHOUT_TEXT_LOG void setTextLog(std::shared_ptr log, int max_priority); #endif @@ -41,7 +41,7 @@ private: /// Previous value of logger element in config. It is used to reinitialize loggers whenever the value changed. std::string config_logger; -#ifdef WITH_TEXT_LOG +#ifndef WITHOUT_TEXT_LOG std::weak_ptr text_log; int text_log_max_priority = -1; #endif diff --git a/src/Loggers/OwnSplitChannel.cpp b/src/Loggers/OwnSplitChannel.cpp index b1502cc4558..35a6d4ad86a 100644 --- a/src/Loggers/OwnSplitChannel.cpp +++ b/src/Loggers/OwnSplitChannel.cpp @@ -21,7 +21,7 @@ namespace DB void OwnSplitChannel::log(const Poco::Message & msg) { -#ifdef WITH_TEXT_LOG +#ifndef WITHOUT_TEXT_LOG auto logs_queue = CurrentThread::getInternalTextLogsQueue(); if (channels.empty() && (logs_queue == nullptr || !logs_queue->isNeeded(msg.getPriority(), msg.getSource()))) @@ -89,7 +89,7 @@ void OwnSplitChannel::logSplit(const Poco::Message & msg) channel.first->log(msg); // ordinary child } -#ifdef WITH_TEXT_LOG +#ifndef WITHOUT_TEXT_LOG auto logs_queue = CurrentThread::getInternalTextLogsQueue(); /// Log to "TCP queue" if message is not too noisy @@ -150,7 +150,7 @@ void OwnSplitChannel::addChannel(Poco::AutoPtr channel, const std channels.emplace(name, ExtendedChannelPtrPair(std::move(channel), dynamic_cast(channel.get()))); } -#ifdef WITH_TEXT_LOG +#ifndef WITHOUT_TEXT_LOG void OwnSplitChannel::addTextLog(std::shared_ptr log, int max_priority) { std::lock_guard lock(text_log_mutex); diff --git a/src/Loggers/OwnSplitChannel.h b/src/Loggers/OwnSplitChannel.h index 16231c105b7..80305c1ccee 100644 --- a/src/Loggers/OwnSplitChannel.h +++ b/src/Loggers/OwnSplitChannel.h @@ -7,7 +7,7 @@ #include #include "ExtendedLogChannel.h" -#ifdef WITH_TEXT_LOG +#ifndef WITHOUT_TEXT_LOG namespace DB { class TextLog; @@ -30,7 +30,7 @@ public: /// Adds a child channel void addChannel(Poco::AutoPtr channel, const std::string & name); -#ifdef WITH_TEXT_LOG +#ifndef WITHOUT_TEXT_LOG void addTextLog(std::shared_ptr log, int max_priority); #endif @@ -47,7 +47,7 @@ private: std::mutex text_log_mutex; -#ifdef WITH_TEXT_LOG +#ifndef WITHOUT_TEXT_LOG std::weak_ptr text_log; std::atomic text_log_max_priority = -1; #endif diff --git a/src/Processors/Formats/IInputFormat.h b/src/Processors/Formats/IInputFormat.h index e2bd208764e..091447e96ee 100644 --- a/src/Processors/Formats/IInputFormat.h +++ b/src/Processors/Formats/IInputFormat.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -55,9 +56,13 @@ public: void addBuffer(std::unique_ptr buffer) { owned_buffers.emplace_back(std::move(buffer)); } + void setErrorsLogger(const InputFormatErrorsLoggerPtr & errors_logger_) { errors_logger = errors_logger_; } + protected: ColumnMappingPtr column_mapping{}; + InputFormatErrorsLoggerPtr errors_logger; + private: /// Number of currently parsed chunk (if parallel parsing is enabled) size_t current_unit_number = 0; diff --git a/src/Processors/Formats/IRowInputFormat.cpp b/src/Processors/Formats/IRowInputFormat.cpp index 3df22002b82..52395338279 100644 --- a/src/Processors/Formats/IRowInputFormat.cpp +++ b/src/Processors/Formats/IRowInputFormat.cpp @@ -52,6 +52,31 @@ IRowInputFormat::IRowInputFormat(Block header, ReadBuffer & in_, Params params_) { } +void IRowInputFormat::logError() +{ + String diagnostic; + String raw_data; + try + { + std::tie(diagnostic, raw_data) = getDiagnosticAndRawData(); + } + catch (const Exception & exception) + { + diagnostic = "Cannot get diagnostic: " + exception.message(); + raw_data = "Cannot get raw data: " + exception.message(); + } + catch (...) + { + /// Error while trying to obtain verbose diagnostic. Ok to ignore. + } + trimLeft(diagnostic, '\n'); + trimRight(diagnostic, '\n'); + + auto now_time = time(nullptr); + + errors_logger->logError(InputFormatErrorsLogger::ErrorEntry{now_time, total_rows, diagnostic, raw_data}); +} + Chunk IRowInputFormat::generate() { if (total_rows == 0) @@ -112,6 +137,9 @@ Chunk IRowInputFormat::generate() if (params.allow_errors_num == 0 && params.allow_errors_ratio == 0) throw; + if (errors_logger) + logError(); + ++num_errors; Float64 current_error_ratio = static_cast(num_errors) / total_rows; diff --git a/src/Processors/Formats/IRowInputFormat.h b/src/Processors/Formats/IRowInputFormat.h index 87caadd93da..a11462549ff 100644 --- a/src/Processors/Formats/IRowInputFormat.h +++ b/src/Processors/Formats/IRowInputFormat.h @@ -65,6 +65,10 @@ protected: /// and collect as much as possible diagnostic information about error. /// If not implemented, returns empty string. virtual std::string getDiagnosticInfo() { return {}; } + /// Get diagnostic info and raw data for a row + virtual std::pair getDiagnosticAndRawData() { return std::make_pair("", ""); } + + void logError(); const BlockMissingValues & getMissingValues() const override { return block_missing_values; } diff --git a/src/Processors/Formats/IRowOutputFormat.cpp b/src/Processors/Formats/IRowOutputFormat.cpp index f2f6b49ed3f..52ce9c1b227 100644 --- a/src/Processors/Formats/IRowOutputFormat.cpp +++ b/src/Processors/Formats/IRowOutputFormat.cpp @@ -40,6 +40,9 @@ void IRowOutputFormat::consume(DB::Chunk chunk) void IRowOutputFormat::consumeTotals(DB::Chunk chunk) { + if (!supportTotals()) + return; + auto num_rows = chunk.getNumRows(); if (num_rows != 1) throw Exception("Got " + toString(num_rows) + " in totals chunk, expected 1", ErrorCodes::LOGICAL_ERROR); @@ -53,6 +56,9 @@ void IRowOutputFormat::consumeTotals(DB::Chunk chunk) void IRowOutputFormat::consumeExtremes(DB::Chunk chunk) { + if (!supportExtremes()) + return; + auto num_rows = chunk.getNumRows(); const auto & columns = chunk.getColumns(); if (num_rows != 2) diff --git a/src/Processors/Formats/IRowOutputFormat.h b/src/Processors/Formats/IRowOutputFormat.h index 7a57753d765..da2eb192e90 100644 --- a/src/Processors/Formats/IRowOutputFormat.h +++ b/src/Processors/Formats/IRowOutputFormat.h @@ -32,6 +32,9 @@ protected: void consumeTotals(Chunk chunk) override; void consumeExtremes(Chunk chunk) override; + virtual bool supportTotals() const { return false; } + virtual bool supportExtremes() const { return false; } + /** Write a row. * Default implementation calls methods to write single values and delimiters * (except delimiter between rows (writeRowBetweenDelimiter())). diff --git a/src/Processors/Formats/ISchemaReader.cpp b/src/Processors/Formats/ISchemaReader.cpp index c7d8b87ab77..9365384f4b7 100644 --- a/src/Processors/Formats/ISchemaReader.cpp +++ b/src/Processors/Formats/ISchemaReader.cpp @@ -113,6 +113,11 @@ NamesAndTypesList IRowSchemaReader::readSchema() "Most likely setting input_format_max_rows_to_read_for_schema_inference is set to 0"); DataTypes data_types = readRowAndGetDataTypes(); + + /// Check that we read at list one column. + if (data_types.empty()) + throw Exception(ErrorCodes::EMPTY_DATA_PASSED, "Cannot read rows from the data"); + /// If column names weren't set, use default names 'c1', 'c2', ... if (column_names.empty()) { @@ -122,9 +127,11 @@ NamesAndTypesList IRowSchemaReader::readSchema() } /// If column names were set, check that the number of names match the number of types. else if (column_names.size() != data_types.size()) + { throw Exception( ErrorCodes::INCORRECT_DATA, "The number of column names {} differs with the number of types {}", column_names.size(), data_types.size()); + } for (size_t i = 0; i != column_names.size(); ++i) { @@ -155,10 +162,6 @@ NamesAndTypesList IRowSchemaReader::readSchema() } } - /// Check that we read at list one column. - if (data_types.empty()) - throw Exception(ErrorCodes::EMPTY_DATA_PASSED, "Cannot read rows from the data"); - NamesAndTypesList result; for (size_t i = 0; i != data_types.size(); ++i) { diff --git a/src/Processors/Formats/Impl/CSVRowOutputFormat.h b/src/Processors/Formats/Impl/CSVRowOutputFormat.h index a36c5ff47fb..0efc00378e5 100644 --- a/src/Processors/Formats/Impl/CSVRowOutputFormat.h +++ b/src/Processors/Formats/Impl/CSVRowOutputFormat.h @@ -34,6 +34,10 @@ private: void writeField(const IColumn & column, const ISerialization & serialization, size_t row_num) override; void writeFieldDelimiter() override; void writeRowEndDelimiter() override; + + bool supportTotals() const override { return true; } + bool supportExtremes() const override { return true; } + void writeBeforeTotals() override; void writeBeforeExtremes() override; diff --git a/src/Processors/Formats/Impl/JSONCompactEachRowRowOutputFormat.h b/src/Processors/Formats/Impl/JSONCompactEachRowRowOutputFormat.h index 63e9e9f1b76..2b4c596d4b5 100644 --- a/src/Processors/Formats/Impl/JSONCompactEachRowRowOutputFormat.h +++ b/src/Processors/Formats/Impl/JSONCompactEachRowRowOutputFormat.h @@ -36,9 +36,8 @@ private: void writeRowStartDelimiter() override; void writeRowEndDelimiter() override; + bool supportTotals() const override { return true; } void consumeTotals(Chunk) override; - /// No extremes. - void consumeExtremes(Chunk) override {} void writeLine(const std::vector & values); diff --git a/src/Processors/Formats/Impl/JSONCompactRowOutputFormat.h b/src/Processors/Formats/Impl/JSONCompactRowOutputFormat.h index d17a6acf019..38123833f10 100644 --- a/src/Processors/Formats/Impl/JSONCompactRowOutputFormat.h +++ b/src/Processors/Formats/Impl/JSONCompactRowOutputFormat.h @@ -31,6 +31,9 @@ private: void writeRowStartDelimiter() override; void writeRowEndDelimiter() override; + bool supportTotals() const override { return true; } + bool supportExtremes() const override { return true; } + void writeBeforeTotals() override; void writeAfterTotals() override; diff --git a/src/Processors/Formats/Impl/JSONEachRowRowOutputFormat.h b/src/Processors/Formats/Impl/JSONEachRowRowOutputFormat.h index ac03c2991bf..37a7029d050 100644 --- a/src/Processors/Formats/Impl/JSONEachRowRowOutputFormat.h +++ b/src/Processors/Formats/Impl/JSONEachRowRowOutputFormat.h @@ -39,10 +39,6 @@ protected: void writePrefix() override; void writeSuffix() override; - /// No totals and extremes. - void consumeTotals(Chunk) override {} - void consumeExtremes(Chunk) override {} - size_t field_number = 0; private: diff --git a/src/Processors/Formats/Impl/JSONRowOutputFormat.h b/src/Processors/Formats/Impl/JSONRowOutputFormat.h index 66b8d0f682e..07c526885f6 100644 --- a/src/Processors/Formats/Impl/JSONRowOutputFormat.h +++ b/src/Processors/Formats/Impl/JSONRowOutputFormat.h @@ -56,6 +56,9 @@ protected: void writeMaxExtreme(const Columns & columns, size_t row_num) override; void writeTotals(const Columns & columns, size_t row_num) override; + bool supportTotals() const override { return true; } + bool supportExtremes() const override { return true; } + void writeBeforeTotals() override; void writeAfterTotals() override; void writeBeforeExtremes() override; diff --git a/src/Processors/Formats/Impl/MsgPackRowInputFormat.cpp b/src/Processors/Formats/Impl/MsgPackRowInputFormat.cpp index b3d237fecfd..bfc4f726edb 100644 --- a/src/Processors/Formats/Impl/MsgPackRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/MsgPackRowInputFormat.cpp @@ -2,6 +2,15 @@ #if USE_MSGPACK +/// FIXME: there is some issue with clang-15, that incorrectly detect a +/// "Attempt to free released memory" in msgpack::unpack(), because of delete +/// operator for zone (from msgpack/v1/detail/cpp11_zone.hpp), hence NOLINT +/// +/// NOTE: that I was not able to suppress it locally, only with +/// NOLINTBEGIN/NOLINTEND +// +// NOLINTBEGIN(clang-analyzer-cplusplus.NewDelete) + #include #include #include @@ -235,8 +244,10 @@ static void insertNull(IColumn & column, DataTypePtr type) assert_cast(column).insertDefault(); } -static void insertUUID(IColumn & column, DataTypePtr /*type*/, const char * value, size_t size) +static void insertUUID(IColumn & column, DataTypePtr type, const char * value, size_t size) { + if (!isUUID(type)) + throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Cannot insert MessagePack UUID into column with type {}.", type->getName()); ReadBufferFromMemory buf(value, size); UUID uuid; readBinaryBigEndian(uuid.toUnderType().items[0], buf); @@ -551,6 +562,8 @@ void registerMsgPackSchemaReader(FormatFactory & factory) } +// NOLINTEND(clang-analyzer-cplusplus.NewDelete) + #else namespace DB diff --git a/src/Processors/Formats/Impl/ORCBlockOutputFormat.cpp b/src/Processors/Formats/Impl/ORCBlockOutputFormat.cpp index 1de2acbb3b9..2b20a4a4bc5 100644 --- a/src/Processors/Formats/Impl/ORCBlockOutputFormat.cpp +++ b/src/Processors/Formats/Impl/ORCBlockOutputFormat.cpp @@ -471,9 +471,8 @@ size_t ORCBlockOutputFormat::getMaxColumnSize(Chunk & chunk) size_t columns_num = chunk.getNumColumns(); size_t max_column_size = 0; for (size_t i = 0; i != columns_num; ++i) - { max_column_size = std::max(max_column_size, getColumnSize(*chunk.getColumns()[i], data_types[i])); - } + return max_column_size; } @@ -481,18 +480,22 @@ void ORCBlockOutputFormat::consume(Chunk chunk) { if (!writer) prepareWriter(); + size_t columns_num = chunk.getNumColumns(); size_t rows_num = chunk.getNumRows(); + /// getMaxColumnSize is needed to write arrays. /// The size of the batch must be no less than total amount of array elements. - ORC_UNIQUE_PTR batch = writer->createRowBatch(getMaxColumnSize(chunk)); + std::unique_ptr batch = writer->createRowBatch(getMaxColumnSize(chunk)); orc::StructVectorBatch & root = dynamic_cast(*batch); + auto columns = chunk.detachColumns(); for (auto & column : columns) column = recursiveRemoveLowCardinality(column); for (size_t i = 0; i != columns_num; ++i) writeColumn(*root.fields[i], *columns[i], data_types[i], nullptr); + root.numElements = rows_num; writer->add(*batch); } diff --git a/src/Processors/Formats/Impl/ORCBlockOutputFormat.h b/src/Processors/Formats/Impl/ORCBlockOutputFormat.h index d4a19353915..6467f2148f5 100644 --- a/src/Processors/Formats/Impl/ORCBlockOutputFormat.h +++ b/src/Processors/Formats/Impl/ORCBlockOutputFormat.h @@ -8,11 +8,13 @@ #include #include + namespace DB { class WriteBuffer; + /// orc::Writer writes only in orc::OutputStream class ORCOutputStream : public orc::OutputStream { @@ -21,7 +23,7 @@ public: uint64_t getLength() const override; uint64_t getNaturalWriteSize() const override; - void write(const void* buf, size_t length) override; + void write(const void * buf, size_t length) override; void close() override {} const std::string& getName() const override { return name; } @@ -31,6 +33,7 @@ private: std::string name = "ORCOutputStream"; }; + class ORCBlockOutputFormat : public IOutputFormat { public: @@ -42,7 +45,7 @@ private: void consume(Chunk chunk) override; void finalizeImpl() override; - ORC_UNIQUE_PTR getORCType(const DataTypePtr & type); + std::unique_ptr getORCType(const DataTypePtr & type); /// ConvertFunc is needed for type UInt8, because firstly UInt8 (char8_t) must be /// converted to unsigned char (bugprone-signed-char-misuse in clang). @@ -75,8 +78,8 @@ private: const FormatSettings format_settings; ORCOutputStream output_stream; DataTypes data_types; - ORC_UNIQUE_PTR writer; - ORC_UNIQUE_PTR schema; + std::unique_ptr writer; + std::unique_ptr schema; orc::WriterOptions options; }; diff --git a/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp b/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp index 318bcaed466..e0693b489bd 100644 --- a/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp +++ b/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp @@ -75,6 +75,7 @@ void ParallelParsingInputFormat::parserThreadFunction(ThreadGroupStatusPtr threa InputFormatPtr input_format = internal_parser_creator(read_buffer); input_format->setCurrentUnitNumber(current_ticket_number); + input_format->setErrorsLogger(errors_logger); InternalParser parser(input_format); unit.chunk_ext.chunk.clear(); diff --git a/src/Processors/Formats/Impl/TSKVRowOutputFormat.h b/src/Processors/Formats/Impl/TSKVRowOutputFormat.h index e9f9071f906..9bc44bbb4d7 100644 --- a/src/Processors/Formats/Impl/TSKVRowOutputFormat.h +++ b/src/Processors/Formats/Impl/TSKVRowOutputFormat.h @@ -22,6 +22,10 @@ private: void writeField(const IColumn & column, const ISerialization & serialization, size_t row_num) override; void writeRowEndDelimiter() override; + /// Disable totals and extremes, because they are enabled in TSV. + bool supportTotals() const override { return false; } + bool supportExtremes() const override { return false; } + NamesAndTypes fields; size_t field_number = 0; }; diff --git a/src/Processors/Formats/Impl/TabSeparatedRowOutputFormat.h b/src/Processors/Formats/Impl/TabSeparatedRowOutputFormat.h index 8781b7be0b1..533e30a3ee1 100644 --- a/src/Processors/Formats/Impl/TabSeparatedRowOutputFormat.h +++ b/src/Processors/Formats/Impl/TabSeparatedRowOutputFormat.h @@ -37,6 +37,10 @@ protected: void writeField(const IColumn & column, const ISerialization & serialization, size_t row_num) override; void writeFieldDelimiter() override final; void writeRowEndDelimiter() override; + + bool supportTotals() const override { return true; } + bool supportExtremes() const override { return true; } + void writeBeforeTotals() override final; void writeBeforeExtremes() override final; diff --git a/src/Processors/Formats/Impl/VerticalRowOutputFormat.h b/src/Processors/Formats/Impl/VerticalRowOutputFormat.h index 796c60d1cb1..c3dbafd8b9b 100644 --- a/src/Processors/Formats/Impl/VerticalRowOutputFormat.h +++ b/src/Processors/Formats/Impl/VerticalRowOutputFormat.h @@ -32,6 +32,9 @@ private: void writeMaxExtreme(const Columns & columns, size_t row_num) override; void writeTotals(const Columns & columns, size_t row_num) override; + bool supportTotals() const override { return true; } + bool supportExtremes() const override { return true; } + void writeBeforeTotals() override; void writeBeforeExtremes() override; diff --git a/src/Processors/Formats/Impl/XMLRowOutputFormat.h b/src/Processors/Formats/Impl/XMLRowOutputFormat.h index abec25efbb9..4d2f34b0bfe 100644 --- a/src/Processors/Formats/Impl/XMLRowOutputFormat.h +++ b/src/Processors/Formats/Impl/XMLRowOutputFormat.h @@ -32,6 +32,9 @@ private: void writeMaxExtreme(const Columns & columns, size_t row_num) override; void writeTotals(const Columns & columns, size_t row_num) override; + bool supportTotals() const override { return true; } + bool supportExtremes() const override { return true; } + void writeBeforeTotals() override; void writeAfterTotals() override; void writeBeforeExtremes() override; diff --git a/src/Processors/Formats/InputFormatErrorsLogger.cpp b/src/Processors/Formats/InputFormatErrorsLogger.cpp new file mode 100644 index 00000000000..e6f8cdd43ee --- /dev/null +++ b/src/Processors/Formats/InputFormatErrorsLogger.cpp @@ -0,0 +1,83 @@ +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ + +namespace +{ + const String DEFAULT_OUTPUT_FORMAT = "CSV"; +} + +InputFormatErrorsLogger::InputFormatErrorsLogger(const ContextPtr & context) +{ + String output_format = context->getSettingsRef().errors_output_format; + if (!FormatFactory::instance().isOutputFormat(output_format)) + output_format = DEFAULT_OUTPUT_FORMAT; + if (context->hasInsertionTable()) + table = context->getInsertionTable().getTableName(); + if (context->getInsertionTable().hasDatabase()) + database = context->getInsertionTable().getDatabaseName(); + + String path_in_setting = context->getSettingsRef().input_format_record_errors_file_path; + errors_file_path = context->getApplicationType() == Context::ApplicationType::SERVER ? context->getUserFilesPath() + path_in_setting + : path_in_setting; + while (fs::exists(errors_file_path)) + { + errors_file_path += "_new"; + } + write_buf = std::make_shared(errors_file_path); + + header = Block{ + {std::make_shared(), "time"}, + {std::make_shared(std::make_shared()), "database"}, + {std::make_shared(std::make_shared()), "table"}, + {std::make_shared(), "offset"}, + {std::make_shared(), "reason"}, + {std::make_shared(), "raw_data"}}; + + writer = context->getOutputFormat(output_format, *write_buf, header); +} + +InputFormatErrorsLogger::~InputFormatErrorsLogger() +{ + writer->finalize(); + writer->flush(); + write_buf->finalize(); +} + +void InputFormatErrorsLogger::logErrorImpl(ErrorEntry entry) +{ + auto error = header.cloneEmpty(); + auto columns = error.mutateColumns(); + columns[0]->insert(entry.time); + database.empty() ? columns[1]->insertDefault() : columns[1]->insert(database); + table.empty() ? columns[2]->insertDefault() : columns[2]->insert(table); + columns[3]->insert(entry.offset); + columns[4]->insert(entry.reason); + columns[5]->insert(entry.raw_data); + error.setColumns(std::move(columns)); + + writer->write(error); +} + +void InputFormatErrorsLogger::logError(ErrorEntry entry) +{ + logErrorImpl(entry); +} + +ParallelInputFormatErrorsLogger::~ParallelInputFormatErrorsLogger() = default; + +void ParallelInputFormatErrorsLogger::logError(ErrorEntry entry) +{ + std::lock_guard lock(write_mutex); + logErrorImpl(entry); +} + +} diff --git a/src/Processors/Formats/InputFormatErrorsLogger.h b/src/Processors/Formats/InputFormatErrorsLogger.h new file mode 100644 index 00000000000..4b3766f4d37 --- /dev/null +++ b/src/Processors/Formats/InputFormatErrorsLogger.h @@ -0,0 +1,54 @@ +#pragma once + +#include +#include + + +namespace DB +{ + +class InputFormatErrorsLogger +{ +public: + struct ErrorEntry + { + time_t time; + size_t offset; + String reason; + String raw_data; + }; + + InputFormatErrorsLogger(const ContextPtr & context); + + virtual ~InputFormatErrorsLogger(); + + virtual void logError(ErrorEntry entry); + void logErrorImpl(ErrorEntry entry); + +private: + Block header; + + String errors_file_path; + std::shared_ptr write_buf; + OutputFormatPtr writer; + + String database; + String table; +}; + +using InputFormatErrorsLoggerPtr = std::shared_ptr; + +class ParallelInputFormatErrorsLogger : public InputFormatErrorsLogger +{ +public: + ParallelInputFormatErrorsLogger(const ContextPtr & context) : InputFormatErrorsLogger(context) { } + + ~ParallelInputFormatErrorsLogger() override; + + void logError(ErrorEntry entry) override; + +private: + std::mutex write_mutex; +}; + +} diff --git a/src/Processors/Formats/RowInputFormatWithDiagnosticInfo.cpp b/src/Processors/Formats/RowInputFormatWithDiagnosticInfo.cpp index f4568830720..35a86bc476d 100644 --- a/src/Processors/Formats/RowInputFormatWithDiagnosticInfo.cpp +++ b/src/Processors/Formats/RowInputFormatWithDiagnosticInfo.cpp @@ -35,12 +35,15 @@ void RowInputFormatWithDiagnosticInfo::updateDiagnosticInfo() offset_of_current_row = in->offset(); } -String RowInputFormatWithDiagnosticInfo::getDiagnosticInfo() +std::pair RowInputFormatWithDiagnosticInfo::getDiagnosticAndRawDataImpl(bool is_errors_record) { - if (in->eof()) - return "Buffer has gone, cannot extract information about what has been parsed."; + WriteBufferFromOwnString out_diag; + WriteBufferFromOwnString out_data; - WriteBufferFromOwnString out; + if (in->eof()) + return std::make_pair( + "Buffer has gone, cannot extract information about what has been parsed.", + "Buffer has gone, cannot extract information about what has been parsed."); const auto & header = getPort().getHeader(); MutableColumns columns = header.cloneEmptyColumns(); @@ -49,8 +52,9 @@ String RowInputFormatWithDiagnosticInfo::getDiagnosticInfo() size_t bytes_read_at_start_of_buffer = in->count() - in->offset(); if (bytes_read_at_start_of_buffer != bytes_read_at_start_of_buffer_on_prev_row) { - out << "Could not print diagnostic info because two last rows aren't in buffer (rare case)\n"; - return out.str(); + out_diag << "Could not print diagnostic info because two last rows aren't in buffer (rare case)\n"; + out_data << "Could not collect raw data because two last rows aren't in buffer (rare case)\n"; + return std::make_pair(out_diag.str(), out_data.str()); } max_length_of_column_name = 0; @@ -65,30 +69,49 @@ String RowInputFormatWithDiagnosticInfo::getDiagnosticInfo() /// Roll back the cursor to the beginning of the previous or current row and parse all over again. But now we derive detailed information. - if (offset_of_prev_row <= in->buffer().size()) + if (!is_errors_record && offset_of_prev_row <= in->buffer().size()) { in->position() = in->buffer().begin() + offset_of_prev_row; - out << "\nRow " << (row_num - 1) << ":\n"; - if (!parseRowAndPrintDiagnosticInfo(columns, out)) - return out.str(); + out_diag << "\nRow " << (row_num - 1) << ":\n"; + if (!parseRowAndPrintDiagnosticInfo(columns, out_diag)) + return std::make_pair(out_diag.str(), out_data.str()); } else { if (in->buffer().size() < offset_of_current_row) { - out << "Could not print diagnostic info because parsing of data hasn't started.\n"; - return out.str(); + out_diag << "Could not print diagnostic info because parsing of data hasn't started.\n"; + out_data << "Could not collect raw data because parsing of data hasn't started.\n"; + return std::make_pair(out_diag.str(), out_data.str()); } in->position() = in->buffer().begin() + offset_of_current_row; } - out << "\nRow " << row_num << ":\n"; - parseRowAndPrintDiagnosticInfo(columns, out); - out << "\n"; + char * data = in->position(); + while (data < in->buffer().end() && *data != '\n' && *data != '\r' && *data != '\0') + { + out_data << *data; + ++data; + } - return out.str(); + out_diag << "\nRow " << row_num << ":\n"; + parseRowAndPrintDiagnosticInfo(columns, out_diag); + out_diag << "\n"; + + return std::make_pair(out_diag.str(), out_data.str()); +} + +String RowInputFormatWithDiagnosticInfo::getDiagnosticInfo() +{ + auto diagnostic_and_raw_data = getDiagnosticAndRawDataImpl(false); + return std::get<0>(diagnostic_and_raw_data); +} + +std::pair RowInputFormatWithDiagnosticInfo::getDiagnosticAndRawData() +{ + return getDiagnosticAndRawDataImpl(true); } bool RowInputFormatWithDiagnosticInfo::deserializeFieldAndPrintDiagnosticInfo(const String & col_name, diff --git a/src/Processors/Formats/RowInputFormatWithDiagnosticInfo.h b/src/Processors/Formats/RowInputFormatWithDiagnosticInfo.h index 5bad24cd482..49793fcd208 100644 --- a/src/Processors/Formats/RowInputFormatWithDiagnosticInfo.h +++ b/src/Processors/Formats/RowInputFormatWithDiagnosticInfo.h @@ -14,7 +14,9 @@ class RowInputFormatWithDiagnosticInfo : public IRowInputFormat public: RowInputFormatWithDiagnosticInfo(const Block & header_, ReadBuffer & in_, const Params & params_); + std::pair getDiagnosticAndRawDataImpl(bool is_errors_record); String getDiagnosticInfo() override; + std::pair getDiagnosticAndRawData() override; void resetParser() override; diff --git a/src/Processors/Sources/ShellCommandSource.h b/src/Processors/Sources/ShellCommandSource.h index a0b4aff4c1b..80ba1d59adf 100644 --- a/src/Processors/Sources/ShellCommandSource.h +++ b/src/Processors/Sources/ShellCommandSource.h @@ -58,7 +58,7 @@ public: /// Pool size valid only if executable_pool = true size_t pool_size = 16; - /// Max command execution time in milliseconds. Valid only if executable_pool = true + /// Max command execution time in seconds. Valid only if executable_pool = true size_t max_command_execution_time_seconds = 10; /// Should pool of processes be created. diff --git a/src/Processors/Transforms/AggregatingTransform.cpp b/src/Processors/Transforms/AggregatingTransform.cpp index 4e55081ca48..ca8e9c0c85b 100644 --- a/src/Processors/Transforms/AggregatingTransform.cpp +++ b/src/Processors/Transforms/AggregatingTransform.cpp @@ -248,8 +248,11 @@ private: throw Exception(ErrorCodes::LOGICAL_ERROR, "Some ready chunks expected"); auto & output = outputs.front(); - output.push(std::move(single_level_chunks.back())); + auto chunk = std::move(single_level_chunks.back()); single_level_chunks.pop_back(); + const auto has_rows = chunk.hasRows(); + if (has_rows) + output.push(std::move(chunk)); if (finished && single_level_chunks.empty()) { @@ -257,7 +260,7 @@ private: return Status::Finished; } - return Status::PortFull; + return has_rows ? Status::PortFull : Status::Ready; } /// Read all sources and try to push current bucket. @@ -281,7 +284,10 @@ private: if (!two_level_chunks[current_bucket_num]) return Status::NeedData; - output.push(std::move(two_level_chunks[current_bucket_num])); + auto chunk = std::move(two_level_chunks[current_bucket_num]); + const auto has_rows = chunk.hasRows(); + if (has_rows) + output.push(std::move(chunk)); ++current_bucket_num; if (current_bucket_num == NUM_BUCKETS) @@ -291,7 +297,7 @@ private: return Status::Finished; } - return Status::PortFull; + return has_rows ? Status::PortFull : Status::Ready; } AggregatingTransformParamsPtr params; diff --git a/src/Processors/Transforms/MongoDBSource.cpp b/src/Processors/Transforms/MongoDBSource.cpp index 19d21f3409e..19289f3f818 100644 --- a/src/Processors/Transforms/MongoDBSource.cpp +++ b/src/Processors/Transforms/MongoDBSource.cpp @@ -250,13 +250,13 @@ namespace if (value.type() == Poco::MongoDB::ElementTraits::TypeId) { std::string string_id = value.toString(); - assert_cast(column).insertDataWithTerminatingZero(string_id.data(), string_id.size() + 1); + assert_cast(column).insertData(string_id.data(), string_id.size()); break; } else if (value.type() == Poco::MongoDB::ElementTraits::TypeId) { String string = static_cast &>(value).value(); - assert_cast(column).insertDataWithTerminatingZero(string.data(), string.size() + 1); + assert_cast(column).insertData(string.data(), string.size()); break; } diff --git a/src/Storages/Distributed/DirectoryMonitor.cpp b/src/Storages/Distributed/DirectoryMonitor.cpp index 16981d26146..e8d48431a9e 100644 --- a/src/Storages/Distributed/DirectoryMonitor.cpp +++ b/src/Storages/Distributed/DirectoryMonitor.cpp @@ -140,6 +140,11 @@ namespace size_t rows = 0; size_t bytes = 0; + UInt32 shard_num = 0; + std::string cluster; + std::string distributed_table; + std::string remote_table; + /// dumpStructure() of the header -- obsolete std::string block_header_string; Block block_header; @@ -195,6 +200,14 @@ namespace in.getFileName(), distributed_header.revision, DBMS_TCP_PROTOCOL_VERSION); } + if (header_buf.hasPendingData()) + { + readVarUInt(distributed_header.shard_num, header_buf); + readStringBinary(distributed_header.cluster, header_buf); + readStringBinary(distributed_header.distributed_table, header_buf); + readStringBinary(distributed_header.remote_table, header_buf); + } + /// Add handling new data here, for example: /// /// if (header_buf.hasPendingData()) @@ -621,18 +634,23 @@ void StorageDistributedDirectoryMonitor::processFile(const std::string & file_pa ReadBufferFromFile in(file_path); const auto & distributed_header = readDistributedHeader(in, log); - auto connection = pool->get(timeouts, &distributed_header.insert_settings); + thread_trace_context = std::make_unique(__PRETTY_FUNCTION__, + distributed_header.client_info.client_trace_context, + this->storage.getContext()->getOpenTelemetrySpanLog()); + thread_trace_context->root_span.addAttribute("clickhouse.shard_num", distributed_header.shard_num); + thread_trace_context->root_span.addAttribute("clickhouse.cluster", distributed_header.cluster); + thread_trace_context->root_span.addAttribute("clickhouse.distributed", distributed_header.distributed_table); + thread_trace_context->root_span.addAttribute("clickhouse.remote", distributed_header.remote_table); + thread_trace_context->root_span.addAttribute("clickhouse.rows", distributed_header.rows); + thread_trace_context->root_span.addAttribute("clickhouse.bytes", distributed_header.bytes); + auto connection = pool->get(timeouts, &distributed_header.insert_settings); LOG_DEBUG(log, "Sending `{}` to {} ({} rows, {} bytes)", file_path, connection->getDescription(), formatReadableQuantity(distributed_header.rows), formatReadableSizeWithBinarySuffix(distributed_header.bytes)); - thread_trace_context = std::make_unique(__PRETTY_FUNCTION__, - distributed_header.client_info.client_trace_context, - this->storage.getContext()->getOpenTelemetrySpanLog()); - RemoteInserter remote{*connection, timeouts, distributed_header.insert_query, distributed_header.insert_settings, diff --git a/src/Storages/Distributed/DistributedSink.cpp b/src/Storages/Distributed/DistributedSink.cpp index ae72fdd84e2..8099a7f2002 100644 --- a/src/Storages/Distributed/DistributedSink.cpp +++ b/src/Storages/Distributed/DistributedSink.cpp @@ -171,7 +171,6 @@ void DistributedSink::writeAsync(const Block & block) } else { - if (storage.getShardingKeyExpr() && (cluster->getShardsInfo().size() > 1)) return writeSplitAsync(block); @@ -291,6 +290,8 @@ DistributedSink::runWritingJob(JobReplica & job, const Block & current_block, si auto thread_group = CurrentThread::getGroup(); return [this, thread_group, &job, ¤t_block, num_shards]() { + OpenTelemetry::SpanHolder span(__PRETTY_FUNCTION__); + if (thread_group) CurrentThread::attachToIfDetached(thread_group); setThreadName("DistrOutStrProc"); @@ -331,15 +332,19 @@ DistributedSink::runWritingJob(JobReplica & job, const Block & current_block, si const Block & shard_block = (num_shards > 1) ? job.current_shard_block : current_block; const Settings & settings = context->getSettingsRef(); - /// Do not initiate INSERT for empty block. size_t rows = shard_block.rows(); + + span.addAttribute("clickhouse.shard_num", shard_info.shard_num); + span.addAttribute("clickhouse.cluster", this->storage.cluster_name); + span.addAttribute("clickhouse.distributed", this->storage.getStorageID().getFullNameNotQuoted()); + span.addAttribute("clickhouse.remote", [this]() { return storage.remote_database + "." + storage.remote_table; }); + span.addAttribute("clickhouse.rows", rows); + span.addAttribute("clickhouse.bytes", [&shard_block]() { return toString(shard_block.bytes()); }); + + /// Do not initiate INSERT for empty block. if (rows == 0) return; - OpenTelemetry::SpanHolder span(__PRETTY_FUNCTION__); - span.addAttribute("clickhouse.shard_num", shard_info.shard_num); - span.addAttribute("clickhouse.written_rows", rows); - if (!job.is_local_job || !settings.prefer_localhost_replica) { if (!job.executor) @@ -610,20 +615,15 @@ void DistributedSink::writeSplitAsync(const Block & block) void DistributedSink::writeAsyncImpl(const Block & block, size_t shard_id) { - OpenTelemetry::SpanHolder span("DistributedSink::writeAsyncImpl()"); - const auto & shard_info = cluster->getShardsInfo()[shard_id]; const auto & settings = context->getSettingsRef(); Block block_to_send = removeSuperfluousColumns(block); - span.addAttribute("clickhouse.shard_num", shard_info.shard_num); - span.addAttribute("clickhouse.written_rows", block.rows()); - if (shard_info.hasInternalReplication()) { if (shard_info.isLocal() && settings.prefer_localhost_replica) /// Prefer insert into current instance directly - writeToLocal(block_to_send, shard_info.getLocalNodeCount()); + writeToLocal(shard_info, block_to_send, shard_info.getLocalNodeCount()); else { const auto & path = shard_info.insertPathForInternalReplication( @@ -631,13 +631,13 @@ void DistributedSink::writeAsyncImpl(const Block & block, size_t shard_id) settings.use_compact_format_in_distributed_parts_names); if (path.empty()) throw Exception("Directory name for async inserts is empty", ErrorCodes::LOGICAL_ERROR); - writeToShard(block_to_send, {path}); + writeToShard(shard_info, block_to_send, {path}); } } else { if (shard_info.isLocal() && settings.prefer_localhost_replica) - writeToLocal(block_to_send, shard_info.getLocalNodeCount()); + writeToLocal(shard_info, block_to_send, shard_info.getLocalNodeCount()); std::vector dir_names; for (const auto & address : cluster->getShardsAddresses()[shard_id]) @@ -645,30 +645,44 @@ void DistributedSink::writeAsyncImpl(const Block & block, size_t shard_id) dir_names.push_back(address.toFullString(settings.use_compact_format_in_distributed_parts_names)); if (!dir_names.empty()) - writeToShard(block_to_send, dir_names); + writeToShard(shard_info, block_to_send, dir_names); } } -void DistributedSink::writeToLocal(const Block & block, size_t repeats) +void DistributedSink::writeToLocal(const Cluster::ShardInfo & shard_info, const Block & block, size_t repeats) { OpenTelemetry::SpanHolder span(__PRETTY_FUNCTION__); - span.addAttribute("db.statement", this->query_string); + span.addAttribute("clickhouse.shard_num", shard_info.shard_num); + span.addAttribute("clickhouse.cluster", this->storage.cluster_name); + span.addAttribute("clickhouse.distributed", this->storage.getStorageID().getFullNameNotQuoted()); + span.addAttribute("clickhouse.remote", [this]() { return storage.remote_database + "." + storage.remote_table; }); + span.addAttribute("clickhouse.rows", [&block]() { return toString(block.rows()); }); + span.addAttribute("clickhouse.bytes", [&block]() { return toString(block.bytes()); }); - InterpreterInsertQuery interp(query_ast, context, allow_materialized); + try + { + InterpreterInsertQuery interp(query_ast, context, allow_materialized); - auto block_io = interp.execute(); - PushingPipelineExecutor executor(block_io.pipeline); + auto block_io = interp.execute(); + PushingPipelineExecutor executor(block_io.pipeline); - executor.start(); - writeBlockConvert(executor, block, repeats, log); - executor.finish(); + executor.start(); + writeBlockConvert(executor, block, repeats, log); + executor.finish(); + } + catch (...) + { + span.addAttribute(std::current_exception()); + throw; + } } -void DistributedSink::writeToShard(const Block & block, const std::vector & dir_names) +void DistributedSink::writeToShard(const Cluster::ShardInfo & shard_info, const Block & block, const std::vector & dir_names) { OpenTelemetry::SpanHolder span(__PRETTY_FUNCTION__); + span.addAttribute("clickhouse.shard_num", shard_info.shard_num); const auto & settings = context->getSettingsRef(); const auto & distributed_settings = storage.getDistributedSettingsRef(); @@ -759,6 +773,11 @@ void DistributedSink::writeToShard(const Block & block, const std::vectorstorage.cluster_name, header_buf); + writeStringBinary(this->storage.getStorageID().getFullNameNotQuoted(), header_buf); + writeStringBinary(this->storage.remote_database + "." + this->storage.remote_table, header_buf); + /// Add new fields here, for example: /// writeVarUInt(my_new_data, header_buf); /// And note that it is safe, because we have checksum and size for header. diff --git a/src/Storages/Distributed/DistributedSink.h b/src/Storages/Distributed/DistributedSink.h index 668cec22e8b..af0c64cbd78 100644 --- a/src/Storages/Distributed/DistributedSink.h +++ b/src/Storages/Distributed/DistributedSink.h @@ -69,9 +69,9 @@ private: Block removeSuperfluousColumns(Block block) const; /// Increments finished_writings_count after each repeat. - void writeToLocal(const Block & block, size_t repeats); + void writeToLocal(const Cluster::ShardInfo & shard_info, const Block & block, size_t repeats); - void writeToShard(const Block & block, const std::vector & dir_names); + void writeToShard(const Cluster::ShardInfo & shard_info, const Block & block, const std::vector & dir_names); /// Performs synchronous insertion to remote nodes. If timeout_exceeded flag was set, throws. diff --git a/src/Storages/HDFS/StorageHDFS.cpp b/src/Storages/HDFS/StorageHDFS.cpp index f93bc45d1a3..45caddb21ea 100644 --- a/src/Storages/HDFS/StorageHDFS.cpp +++ b/src/Storages/HDFS/StorageHDFS.cpp @@ -255,7 +255,7 @@ private: class HDFSSource::URISIterator::Impl { public: - explicit Impl(const std::vector & uris_, ContextPtr context) + explicit Impl(const std::vector & uris_, ContextPtr context) { auto path_and_uri = getPathFromUriAndUriWithoutPath(uris_[0]); HDFSBuilderWrapper builder = createHDFSBuilder(path_and_uri.second + "/", context->getGlobalContext()->getConfigRef()); @@ -293,7 +293,7 @@ String HDFSSource::DisclosedGlobIterator::next() return pimpl->next(); } -HDFSSource::URISIterator::URISIterator(const std::vector & uris_, ContextPtr context) +HDFSSource::URISIterator::URISIterator(const std::vector & uris_, ContextPtr context) : pimpl(std::make_shared(uris_, context)) { } diff --git a/src/Storages/HDFS/StorageHDFS.h b/src/Storages/HDFS/StorageHDFS.h index a0d61f4bd2a..896371f9685 100644 --- a/src/Storages/HDFS/StorageHDFS.h +++ b/src/Storages/HDFS/StorageHDFS.h @@ -86,7 +86,7 @@ private: const String & format_name, const ContextPtr & ctx); - std::vector uris; + std::vector uris; String format_name; String compression_method; const bool distributed_processing; @@ -116,7 +116,7 @@ public: class URISIterator { public: - URISIterator(const std::vector & uris_, ContextPtr context); + URISIterator(const std::vector & uris_, ContextPtr context); String next(); private: class Impl; diff --git a/src/Storages/Kafka/StorageKafka.cpp b/src/Storages/Kafka/StorageKafka.cpp index 06ce4fb308d..d9bacffd053 100644 --- a/src/Storages/Kafka/StorageKafka.cpp +++ b/src/Storages/Kafka/StorageKafka.cpp @@ -179,8 +179,6 @@ namespace void loadFromConfig(cppkafka::Configuration & conf, const Poco::Util::AbstractConfiguration & config, const std::string & path) { Poco::Util::AbstractConfiguration::Keys keys; - std::vector errstr(512); - config.keys(path, keys); for (const auto & key : keys) diff --git a/src/Storages/MergeTree/DataPartStorageOnDisk.cpp b/src/Storages/MergeTree/DataPartStorageOnDisk.cpp index 894eec12f0c..5245bc89e0c 100644 --- a/src/Storages/MergeTree/DataPartStorageOnDisk.cpp +++ b/src/Storages/MergeTree/DataPartStorageOnDisk.cpp @@ -204,13 +204,12 @@ DataPartStorageBuilderPtr DataPartStorageOnDisk::getBuilder() const } void DataPartStorageOnDisk::remove( - bool can_remove_shared_data, - const NameSet & names_not_to_remove, + CanRemoveCallback && can_remove_callback, const MergeTreeDataPartChecksums & checksums, std::list projections, bool is_temp, MergeTreeDataPartState state, - Poco::Logger * log) const + Poco::Logger * log) { /// NOTE We rename part to delete_tmp_ instead of delete_tmp_ to avoid race condition /// when we try to remove two parts with the same name, but different relative paths, @@ -239,13 +238,16 @@ void DataPartStorageOnDisk::remove( fs::path to = fs::path(root_path) / part_dir_without_slash; + std::optional can_remove_description; + auto disk = volume->getDisk(); if (disk->exists(to)) { LOG_WARNING(log, "Directory {} (to which part must be renamed before removing) already exists. Most likely this is due to unclean restart or race condition. Removing it.", fullPath(disk, to)); try { - disk->removeSharedRecursive(fs::path(to) / "", !can_remove_shared_data, names_not_to_remove); + can_remove_description.emplace(can_remove_callback()); + disk->removeSharedRecursive(fs::path(to) / "", !can_remove_description->can_remove_anything, can_remove_description->files_not_to_remove); } catch (...) { @@ -257,6 +259,7 @@ void DataPartStorageOnDisk::remove( try { disk->moveDirectory(from, to); + onRename(root_path, part_dir_without_slash); } catch (const fs::filesystem_error & e) { @@ -268,6 +271,9 @@ void DataPartStorageOnDisk::remove( throw; } + if (!can_remove_description) + can_remove_description.emplace(can_remove_callback()); + // Record existing projection directories so we don't remove them twice std::unordered_set projection_directories; std::string proj_suffix = ".proj"; @@ -278,7 +284,7 @@ void DataPartStorageOnDisk::remove( clearDirectory( fs::path(to) / proj_dir_name, - can_remove_shared_data, names_not_to_remove, projection.checksums, {}, is_temp, state, log, true); + can_remove_description->can_remove_anything, can_remove_description->files_not_to_remove, projection.checksums, {}, is_temp, state, log, true); } /// It is possible that we are removing the part which have a written but not loaded projection. @@ -305,7 +311,7 @@ void DataPartStorageOnDisk::remove( clearDirectory( fs::path(to) / name, - can_remove_shared_data, names_not_to_remove, tmp_checksums, {}, is_temp, state, log, true); + can_remove_description->can_remove_anything, can_remove_description->files_not_to_remove, tmp_checksums, {}, is_temp, state, log, true); } catch (...) { @@ -315,7 +321,7 @@ void DataPartStorageOnDisk::remove( } } - clearDirectory(to, can_remove_shared_data, names_not_to_remove, checksums, projection_directories, is_temp, state, log, false); + clearDirectory(to, can_remove_description->can_remove_anything, can_remove_description->files_not_to_remove, checksums, projection_directories, is_temp, state, log, false); } void DataPartStorageOnDisk::clearDirectory( @@ -348,18 +354,11 @@ void DataPartStorageOnDisk::clearDirectory( /// Remove each expected file in directory, then remove directory itself. RemoveBatchRequest request; -#if !defined(__clang__) -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wunused-variable" -#endif for (const auto & [file, _] : checksums.files) { if (skip_directories.find(file) == skip_directories.end()) request.emplace_back(fs::path(dir) / file); } -#if !defined(__clang__) -# pragma GCC diagnostic pop -#endif for (const auto & file : {"checksums.txt", "columns.txt"}) request.emplace_back(fs::path(dir) / file); diff --git a/src/Storages/MergeTree/DataPartStorageOnDisk.h b/src/Storages/MergeTree/DataPartStorageOnDisk.h index f02ef26f811..51b557767d4 100644 --- a/src/Storages/MergeTree/DataPartStorageOnDisk.h +++ b/src/Storages/MergeTree/DataPartStorageOnDisk.h @@ -45,13 +45,12 @@ public: void checkConsistency(const MergeTreeDataPartChecksums & checksums) const override; void remove( - bool can_remove_shared_data, - const NameSet & names_not_to_remove, + CanRemoveCallback && can_remove_callback, const MergeTreeDataPartChecksums & checksums, std::list projections, bool is_temp, MergeTreeDataPartState state, - Poco::Logger * log) const override; + Poco::Logger * log) override; std::string getRelativePathForPrefix(Poco::Logger * log, const String & prefix, bool detached) const override; diff --git a/src/Storages/MergeTree/IDataPartStorage.h b/src/Storages/MergeTree/IDataPartStorage.h index 9da8a5eae03..bd449d46075 100644 --- a/src/Storages/MergeTree/IDataPartStorage.h +++ b/src/Storages/MergeTree/IDataPartStorage.h @@ -12,6 +12,13 @@ namespace DB class ReadBufferFromFileBase; class WriteBufferFromFileBase; +struct CanRemoveDescription +{ + bool can_remove_anything; + NameSet files_not_to_remove; + +}; +using CanRemoveCallback = std::function; class IDataPartStorageIterator { @@ -113,13 +120,12 @@ public: /// can_remove_shared_data, names_not_to_remove are specific for DiskObjectStorage. /// projections, checksums are needed to avoid recursive listing virtual void remove( - bool can_remove_shared_data, - const NameSet & names_not_to_remove, + CanRemoveCallback && can_remove_callback, const MergeTreeDataPartChecksums & checksums, std::list projections, bool is_temp, MergeTreeDataPartState state, - Poco::Logger * log) const = 0; + Poco::Logger * log) = 0; /// Get a name like 'prefix_partdir_tryN' which does not exist in a root dir. /// TODO: remove it. diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index bafbb31f3a2..e4c54b933da 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -1297,7 +1297,7 @@ catch (Exception & e) bool IMergeTreeDataPart::wasInvolvedInTransaction() const { - assert(!version.creation_tid.isEmpty() || (state == MergeTreeDataPartState::Temporary /* && std::uncaught_exceptions() */)); + assert(!storage.data_parts_loading_finished || !version.creation_tid.isEmpty() || (state == MergeTreeDataPartState::Temporary /* && std::uncaught_exceptions() */)); bool created_by_transaction = !version.creation_tid.isPrehistoric(); bool removed_by_transaction = version.isRemovalTIDLocked() && version.removal_tid_lock != Tx::PrehistoricTID.getHash(); return created_by_transaction || removed_by_transaction; @@ -1448,13 +1448,18 @@ void IMergeTreeDataPart::remove() const assert(assertHasValidVersionMetadata()); part_is_probably_removed_from_disk = true; - auto [can_remove, files_not_to_remove] = canRemovePart(); + auto can_remove_callback = [this] () + { + auto [can_remove, files_not_to_remove] = canRemovePart(); + if (!can_remove) + LOG_TRACE(storage.log, "Blobs of part {} cannot be removed", name); - if (!can_remove) - LOG_TRACE(storage.log, "Blobs of part {} cannot be removed", name); + if (!files_not_to_remove.empty()) + LOG_TRACE(storage.log, "Some blobs ({}) of part {} cannot be removed", fmt::join(files_not_to_remove, ", "), name); + + return CanRemoveDescription{.can_remove_anything = can_remove, .files_not_to_remove = files_not_to_remove }; + }; - if (!files_not_to_remove.empty()) - LOG_TRACE(storage.log, "Some blobs ({}) of part {} cannot be removed", fmt::join(files_not_to_remove, ", "), name); if (!isStoredOnDisk()) return; @@ -1474,7 +1479,7 @@ void IMergeTreeDataPart::remove() const projection_checksums.emplace_back(IDataPartStorage::ProjectionChecksums{.name = p_name, .checksums = projection_part->checksums}); } - data_part_storage->remove(can_remove, files_not_to_remove, checksums, projection_checksums, is_temp, getState(), storage.log); + data_part_storage->remove(std::move(can_remove_callback), checksums, projection_checksums, is_temp, getState(), storage.log); } String IMergeTreeDataPart::getRelativePathForPrefix(const String & prefix, bool detached) const diff --git a/src/Storages/MergeTree/KeyCondition.cpp b/src/Storages/MergeTree/KeyCondition.cpp index f5eeb4ed35c..2eea7a2f359 100644 --- a/src/Storages/MergeTree/KeyCondition.cpp +++ b/src/Storages/MergeTree/KeyCondition.cpp @@ -116,10 +116,20 @@ static void appendColumnNameWithoutAlias(const ActionsDAG::Node & node, WriteBuf { switch (node.type) { - case (ActionsDAG::ActionType::INPUT): [[fallthrough]]; - case (ActionsDAG::ActionType::COLUMN): + case (ActionsDAG::ActionType::INPUT): writeString(node.result_name, out); break; + case (ActionsDAG::ActionType::COLUMN): + { + /// If it was created from ASTLiteral, then result_name can be an alias. + /// We need to convert value back to string here. + if (const auto * column_const = typeid_cast(node.column.get())) + writeString(applyVisitor(FieldVisitorToString(), column_const->getField()), out); + /// It may be possible that column is ColumnSet + else + writeString(node.result_name, out); + break; + } case (ActionsDAG::ActionType::ALIAS): appendColumnNameWithoutAlias(*node.children.front(), out, legacy); break; diff --git a/src/Storages/MergeTree/MergeFromLogEntryTask.cpp b/src/Storages/MergeTree/MergeFromLogEntryTask.cpp index d51cd6aa07d..9bae4a840bb 100644 --- a/src/Storages/MergeTree/MergeFromLogEntryTask.cpp +++ b/src/Storages/MergeTree/MergeFromLogEntryTask.cpp @@ -223,6 +223,27 @@ ReplicatedMergeMutateTaskBase::PrepareResult MergeFromLogEntryTask::prepare() .part_log_writer = {} }; } + else if (!storage.findReplicaHavingCoveringPart(entry.new_part_name, /* active */ false, dummy).empty()) + { + /// Why this if still needed? We can check for part in zookeeper, don't find it and sleep for any amount of time. During this sleep part will be actually committed from other replica + /// and exclusive zero copy lock will be released. We will take the lock and execute merge one more time, while it was possible just to download the part from other replica. + /// + /// It's also possible just because reads in [Zoo]Keeper are not lineariazable. + /// + /// NOTE: In case of mutation and hardlinks it can even lead to extremely rare dataloss (we will produce new part with the same hardlinks, don't fetch the same from other replica), so this check is important. + zero_copy_lock->lock->unlock(); + + LOG_DEBUG(log, "We took zero copy lock, but merge of part {} finished by some other replica, will release lock and download merged part to avoid data duplication", entry.new_part_name); + return PrepareResult{ + .prepared_successfully = false, + .need_to_check_missing_part_in_fetch = true, + .part_log_writer = {} + }; + } + else + { + LOG_DEBUG(log, "Zero copy lock taken, will merge part {}", entry.new_part_name); + } } } diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 73da9d362b8..f70577dc9d4 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -27,6 +27,7 @@ #include #include #include +#include #include #include #include @@ -1561,6 +1562,7 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks) calculateColumnAndSecondaryIndexSizesImpl(); LOG_DEBUG(log, "Loaded data parts ({} items)", data_parts_indexes.size()); + data_parts_loading_finished = true; } /// Is the part directory old. @@ -1635,11 +1637,10 @@ size_t MergeTreeData::clearOldTemporaryDirectories(size_t custom_directories_lif /// We don't control the amount of refs for temporary parts so we cannot decide can we remove blobs /// or not. So we are not doing it bool keep_shared = false; - if (it->path().find("fetch") != std::string::npos) + if (disk->supportZeroCopyReplication() && settings->allow_remote_fs_zero_copy_replication) { - keep_shared = disk->supportZeroCopyReplication() && settings->allow_remote_fs_zero_copy_replication; - if (keep_shared) - LOG_WARNING(log, "Since zero-copy replication is enabled we are not going to remove blobs from shared storage for {}", full_path); + LOG_WARNING(log, "Since zero-copy replication is enabled we are not going to remove blobs from shared storage for {}", full_path); + keep_shared = true; } disk->removeSharedRecursive(it->path(), keep_shared, {}); @@ -2135,6 +2136,7 @@ void MergeTreeData::renameInMemory(const StorageID & new_table_id) void MergeTreeData::dropAllData() { LOG_TRACE(log, "dropAllData: waiting for locks."); + auto settings_ptr = getSettings(); auto lock = lockParts(); diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index 4158517fc23..94bca094a86 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -1034,6 +1034,8 @@ protected: /// True if at least one part was created/removed with transaction. mutable std::atomic_bool transactions_enabled = false; + std::atomic_bool data_parts_loading_finished = false; + /// Work with data parts struct TagByInfo{}; @@ -1242,6 +1244,9 @@ protected: /// Attaches restored parts to the storage. virtual void attachRestoredParts(MutableDataPartsVector && parts) = 0; + void resetObjectColumnsFromActiveParts(const DataPartsLock & lock); + void updateObjectColumns(const DataPartPtr & part, const DataPartsLock & lock); + static void incrementInsertedPartsProfileEvent(MergeTreeDataPartType type); static void incrementMergedPartsProfileEvent(MergeTreeDataPartType type); @@ -1329,9 +1334,6 @@ private: DataPartsVector & duplicate_parts_to_remove, MutableDataPartsVector & parts_from_wal); - void resetObjectColumnsFromActiveParts(const DataPartsLock & lock); - void updateObjectColumns(const DataPartPtr & part, const DataPartsLock & lock); - /// Create zero-copy exclusive lock for part and disk. Useful for coordination of /// distributed operations which can lead to data duplication. Implemented only in ReplicatedMergeTree. virtual std::optional tryCreateZeroCopyExclusiveLock(const String &, const DiskPtr &) { return std::nullopt; } diff --git a/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/src/Storages/MergeTree/MergeTreeDataWriter.cpp index 97900eef22b..95faef6aac7 100644 --- a/src/Storages/MergeTree/MergeTreeDataWriter.cpp +++ b/src/Storages/MergeTree/MergeTreeDataWriter.cpp @@ -483,16 +483,6 @@ MergeTreeDataWriter::TemporaryPart MergeTreeDataWriter::writeTempPart( return temp_part; } -void MergeTreeDataWriter::deduceTypesOfObjectColumns(const StorageSnapshotPtr & storage_snapshot, Block & block) -{ - if (!storage_snapshot->object_columns.empty()) - { - auto options = GetColumnsOptions(GetColumnsOptions::AllPhysical).withExtendedObjects(); - auto storage_columns = storage_snapshot->getColumns(options); - convertObjectsToTuples(block, storage_columns); - } -} - MergeTreeDataWriter::TemporaryPart MergeTreeDataWriter::writeProjectionPartImpl( const String & part_name, MergeTreeDataPartType part_type, diff --git a/src/Storages/MergeTree/MergeTreeDataWriter.h b/src/Storages/MergeTree/MergeTreeDataWriter.h index 2f9ab1aae8b..00438a29fa1 100644 --- a/src/Storages/MergeTree/MergeTreeDataWriter.h +++ b/src/Storages/MergeTree/MergeTreeDataWriter.h @@ -45,8 +45,6 @@ public: */ static BlocksWithPartition splitBlockIntoParts(const Block & block, size_t max_parts, const StorageMetadataPtr & metadata_snapshot, ContextPtr context); - static void deduceTypesOfObjectColumns(const StorageSnapshotPtr & storage_snapshot, Block & block); - /// This structure contains not completely written temporary part. /// Some writes may happen asynchronously, e.g. for blob storages. /// You should call finalize() to wait until all data is written. diff --git a/src/Storages/MergeTree/MergeTreeIndexConditionBloomFilter.cpp b/src/Storages/MergeTree/MergeTreeIndexConditionBloomFilter.cpp index 9d1a075a63f..dc890910224 100644 --- a/src/Storages/MergeTree/MergeTreeIndexConditionBloomFilter.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexConditionBloomFilter.cpp @@ -242,11 +242,21 @@ bool MergeTreeIndexConditionBloomFilter::traverseAtomAST(const ASTPtr & node, Bl DataTypePtr const_type; if (KeyCondition::getConstant(node, block_with_constants, const_value, const_type)) { - if (const_value.getType() == Field::Types::UInt64 || const_value.getType() == Field::Types::Int64 || - const_value.getType() == Field::Types::Float64) + if (const_value.getType() == Field::Types::UInt64) { - /// Zero in all types is represented in memory the same way as in UInt64. - out.function = const_value.reinterpret() ? RPNElement::ALWAYS_TRUE : RPNElement::ALWAYS_FALSE; + out.function = const_value.get() ? RPNElement::ALWAYS_TRUE : RPNElement::ALWAYS_FALSE; + return true; + } + + if (const_value.getType() == Field::Types::Int64) + { + out.function = const_value.get() ? RPNElement::ALWAYS_TRUE : RPNElement::ALWAYS_FALSE; + return true; + } + + if (const_value.getType() == Field::Types::Float64) + { + out.function = const_value.get() ? RPNElement::ALWAYS_TRUE : RPNElement::ALWAYS_FALSE; return true; } } diff --git a/src/Storages/MergeTree/MergeTreeSink.cpp b/src/Storages/MergeTree/MergeTreeSink.cpp index 5eaa8ec8004..5d00db861a8 100644 --- a/src/Storages/MergeTree/MergeTreeSink.cpp +++ b/src/Storages/MergeTree/MergeTreeSink.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include namespace ProfileEvents @@ -56,7 +57,7 @@ void MergeTreeSink::consume(Chunk chunk) { auto block = getHeader().cloneWithColumns(chunk.detachColumns()); - storage.writer.deduceTypesOfObjectColumns(storage_snapshot, block); + deduceTypesOfObjectColumns(storage_snapshot, block); auto part_blocks = storage.writer.splitBlockIntoParts(block, max_parts_per_block, metadata_snapshot, context); using DelayedPartitions = std::vector; diff --git a/src/Storages/MergeTree/MutateFromLogEntryTask.cpp b/src/Storages/MergeTree/MutateFromLogEntryTask.cpp index a51eb7854ab..fc8b22865c4 100644 --- a/src/Storages/MergeTree/MutateFromLogEntryTask.cpp +++ b/src/Storages/MergeTree/MutateFromLogEntryTask.cpp @@ -134,6 +134,29 @@ ReplicatedMergeMutateTaskBase::PrepareResult MutateFromLogEntryTask::prepare() .part_log_writer = {} }; } + else if (!storage.findReplicaHavingCoveringPart(entry.new_part_name, /* active */ false, dummy).empty()) + { + /// Why this if still needed? We can check for part in zookeeper, don't find it and sleep for any amount of time. During this sleep part will be actually committed from other replica + /// and exclusive zero copy lock will be released. We will take the lock and execute mutation one more time, while it was possible just to download the part from other replica. + /// + /// It's also possible just because reads in [Zoo]Keeper are not lineariazable. + /// + /// NOTE: In case of mutation and hardlinks it can even lead to extremely rare dataloss (we will produce new part with the same hardlinks, don't fetch the same from other replica), so this check is important. + /// + /// In case of DROP_RANGE on fast replica and stale replica we can have some failed select queries in case of zero copy replication. + zero_copy_lock->lock->unlock(); + + LOG_DEBUG(log, "We took zero copy lock, but mutation of part {} finished by some other replica, will release lock and download mutated part to avoid data duplication", entry.new_part_name); + return PrepareResult{ + .prepared_successfully = false, + .need_to_check_missing_part_in_fetch = true, + .part_log_writer = {} + }; + } + else + { + LOG_DEBUG(log, "Zero copy lock taken, will mutate part {}", entry.new_part_name); + } } } diff --git a/src/Storages/MergeTree/MutateFromLogEntryTask.h b/src/Storages/MergeTree/MutateFromLogEntryTask.h index a0bbaabda85..416b0c92522 100644 --- a/src/Storages/MergeTree/MutateFromLogEntryTask.h +++ b/src/Storages/MergeTree/MutateFromLogEntryTask.h @@ -6,6 +6,7 @@ #include #include #include +#include namespace DB { @@ -18,7 +19,12 @@ public: ReplicatedMergeTreeQueue::SelectedEntryPtr selected_entry_, StorageReplicatedMergeTree & storage_, Callback && task_result_callback_) - : ReplicatedMergeMutateTaskBase(&Poco::Logger::get("MutateFromLogEntryTask"), storage_, selected_entry_, task_result_callback_) {} + : ReplicatedMergeMutateTaskBase( + &Poco::Logger::get(storage_.getStorageID().getShortName() + "::" + selected_entry_->log_entry->new_part_name + "(MutateFromLogEntryTask)"), + storage_, + selected_entry_, + task_result_callback_) + {} UInt64 getPriority() override { return priority; } diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp index 6c7fbcb52d8..b9bd027cde2 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include @@ -161,7 +162,7 @@ void ReplicatedMergeTreeSink::consume(Chunk chunk) */ size_t replicas_num = checkQuorumPrecondition(zookeeper); - storage.writer.deduceTypesOfObjectColumns(storage_snapshot, block); + deduceTypesOfObjectColumns(storage_snapshot, block); auto part_blocks = storage.writer.splitBlockIntoParts(block, max_parts_per_block, metadata_snapshot, context); using DelayedPartitions = std::vector; @@ -203,11 +204,11 @@ void ReplicatedMergeTreeSink::consume(Chunk chunk) } block_id = temp_part.part->getZeroLevelPartBlockID(block_dedup_token); - LOG_DEBUG(log, "Wrote block with ID '{}', {} rows on {} replicas", block_id, current_block.block.rows(), replicas_num); + LOG_DEBUG(log, "Wrote block with ID '{}', {} rows{}", block_id, current_block.block.rows(), quorumLogMessage(replicas_num)); } else { - LOG_DEBUG(log, "Wrote block with {} rows on {} replicas", current_block.block.rows(), replicas_num); + LOG_DEBUG(log, "Wrote block with {} rows{}", current_block.block.rows(), quorumLogMessage(replicas_num)); } UInt64 elapsed_ns = watch.elapsed(); @@ -639,7 +640,7 @@ void ReplicatedMergeTreeSink::waitForQuorum( size_t replicas_num) const { /// We are waiting for quorum to be satisfied. - LOG_TRACE(log, "Waiting for quorum '{}' for part {} on {} replicas", quorum_path, part_name, replicas_num); + LOG_TRACE(log, "Waiting for quorum '{}' for part {}{}", quorum_path, part_name, quorumLogMessage(replicas_num)); try { @@ -684,6 +685,13 @@ void ReplicatedMergeTreeSink::waitForQuorum( LOG_TRACE(log, "Quorum '{}' for part {} satisfied", quorum_path, part_name); } +String ReplicatedMergeTreeSink::quorumLogMessage(size_t replicas_num) const +{ + if (!isQuorumEnabled()) + return ""; + return fmt::format(" (quorum {} of {} replicas)", getQuorumSize(replicas_num), replicas_num); +} + size_t ReplicatedMergeTreeSink::getQuorumSize(size_t replicas_num) const { if (!isQuorumEnabled()) diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeSink.h b/src/Storages/MergeTree/ReplicatedMergeTreeSink.h index 48e94ef5659..ab729e6edec 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeSink.h +++ b/src/Storages/MergeTree/ReplicatedMergeTreeSink.h @@ -96,6 +96,7 @@ private: size_t getQuorumSize(size_t replicas_num) const; bool isQuorumEnabled() const; + String quorumLogMessage(size_t replicas_num) const; /// Used in logs for debug purposes size_t quorum_timeout_ms; size_t max_parts_per_block; diff --git a/src/Storages/SelectQueryInfo.h b/src/Storages/SelectQueryInfo.h index c41b422199d..f2835ab4dbf 100644 --- a/src/Storages/SelectQueryInfo.h +++ b/src/Storages/SelectQueryInfo.h @@ -117,10 +117,10 @@ struct InputOrderInfo * sort_description_for_merging will be equal to (c, d) and * used_prefix_of_sorting_key_size will be equal to 4. */ - size_t used_prefix_of_sorting_key_size; + const size_t used_prefix_of_sorting_key_size; - int direction; - UInt64 limit; + const int direction; + const UInt64 limit; InputOrderInfo( const SortDescription & sort_description_for_merging_, diff --git a/src/Storages/StorageKeeperMap.cpp b/src/Storages/StorageKeeperMap.cpp new file mode 100644 index 00000000000..3ae7cf7a7e4 --- /dev/null +++ b/src/Storages/StorageKeeperMap.cpp @@ -0,0 +1,771 @@ +#include + +#include + +#include + +#include +#include +#include + +#include + +#include + +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; + extern const int BAD_ARGUMENTS; + extern const int KEEPER_EXCEPTION; + extern const int LOGICAL_ERROR; + extern const int LIMIT_EXCEEDED; +} + +namespace +{ + +std::string formattedAST(const ASTPtr & ast) +{ + if (!ast) + return ""; + return serializeAST(*ast); +} + +void verifyTableId(const StorageID & table_id) +{ + if (!table_id.hasUUID()) + { + auto database = DatabaseCatalog::instance().getDatabase(table_id.database_name); + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "KeeperMap cannot be used with '{}' database because it uses {} engine. Please use Atomic or Replicated database", + table_id.getDatabaseName(), + database->getEngineName()); + } + +} + +} + +class StorageKeeperMapSink : public SinkToStorage +{ + StorageKeeperMap & storage; + std::unordered_map new_values; + size_t primary_key_pos; + +public: + StorageKeeperMapSink(StorageKeeperMap & storage_, const StorageMetadataPtr & metadata_snapshot) + : SinkToStorage(metadata_snapshot->getSampleBlock()), storage(storage_) + { + auto primary_key = storage.getPrimaryKey(); + assert(primary_key.size() == 1); + primary_key_pos = getHeader().getPositionByName(primary_key[0]); + } + + std::string getName() const override { return "StorageKeeperMapSink"; } + + void consume(Chunk chunk) override + { + auto rows = chunk.getNumRows(); + auto block = getHeader().cloneWithColumns(chunk.detachColumns()); + + WriteBufferFromOwnString wb_key; + WriteBufferFromOwnString wb_value; + + for (size_t i = 0; i < rows; ++i) + { + wb_key.restart(); + wb_value.restart(); + + size_t idx = 0; + for (const auto & elem : block) + { + elem.type->getDefaultSerialization()->serializeBinary(*elem.column, i, idx == primary_key_pos ? wb_key : wb_value); + ++idx; + } + + auto key = base64Encode(wb_key.str(), /* url_encoding */ true); + new_values[std::move(key)] = std::move(wb_value.str()); + } + } + + void onFinish() override + { + auto zookeeper = storage.getClient(); + + Coordination::Requests requests; + + auto keys_limit = storage.keysLimit(); + + size_t current_keys_num = 0; + size_t new_keys_num = 0; + + // We use keys limit as a soft limit so we ignore some cases when it can be still exceeded + // (e.g if parallel insert queries are being run) + if (keys_limit != 0) + { + Coordination::Stat data_stat; + zookeeper->get(storage.dataPath(), &data_stat); + current_keys_num = data_stat.numChildren; + } + + std::vector>> exist_responses; + for (const auto & [key, value] : new_values) + { + auto path = storage.fullPathForKey(key); + + exist_responses.push_back({&key, zookeeper->asyncExists(path)}); + } + + for (auto & [key, response] : exist_responses) + { + if (response.get().error == Coordination::Error::ZOK) + { + requests.push_back(zkutil::makeSetRequest(storage.fullPathForKey(*key), new_values[*key], -1)); + } + else + { + requests.push_back(zkutil::makeCreateRequest(storage.fullPathForKey(*key), new_values[*key], zkutil::CreateMode::Persistent)); + ++new_keys_num; + } + } + + if (new_keys_num != 0) + { + auto will_be = current_keys_num + new_keys_num; + if (keys_limit != 0 && will_be > keys_limit) + throw Exception( + ErrorCodes::LIMIT_EXCEEDED, + "Limit would be exceeded by inserting {} new key(s). Limit is {}, while the number of keys would be {}", + new_keys_num, + keys_limit, + will_be); + } + + zookeeper->multi(requests); + } +}; + +template +class StorageKeeperMapSource : public ISource +{ + const StorageKeeperMap & storage; + size_t max_block_size; + + using KeyContainerPtr = std::shared_ptr; + KeyContainerPtr container; + using KeyContainerIter = typename KeyContainer::const_iterator; + KeyContainerIter it; + KeyContainerIter end; + +public: + StorageKeeperMapSource( + const StorageKeeperMap & storage_, + const Block & header, + size_t max_block_size_, + KeyContainerPtr container_, + KeyContainerIter begin_, + KeyContainerIter end_) + : ISource(header), storage(storage_), max_block_size(max_block_size_), container(std::move(container_)), it(begin_), end(end_) + { + } + + std::string getName() const override { return "StorageKeeperMapSource"; } + + Chunk generate() override + { + if (it >= end) + { + it = {}; + return {}; + } + + using KeyType = typename KeyContainer::value_type; + if constexpr (std::same_as) + { + const auto & sample_block = getPort().getHeader(); + const auto & key_column_type = sample_block.getByName(storage.getPrimaryKey().at(0)).type; + auto raw_keys = serializeKeysToRawString(it, end, key_column_type, max_block_size); + + for (auto & raw_key : raw_keys) + raw_key = base64Encode(raw_key, /* url_encoding */ true); + + return storage.getBySerializedKeys(raw_keys, nullptr); + } + else + { + size_t elem_num = std::min(max_block_size, static_cast(end - it)); + auto chunk = storage.getBySerializedKeys(std::span{it, it + elem_num}, nullptr); + it += elem_num; + return chunk; + } + } +}; + +StorageKeeperMap::StorageKeeperMap( + ContextPtr context_, + const StorageID & table_id, + const StorageInMemoryMetadata & metadata, + bool attach, + std::string_view primary_key_, + const std::string & root_path_, + UInt64 keys_limit_) + : IStorage(table_id) + , WithContext(context_->getGlobalContext()) + , root_path(zkutil::extractZooKeeperPath(root_path_, false)) + , primary_key(primary_key_) + , zookeeper_name(zkutil::extractZooKeeperName(root_path_)) + , keys_limit(keys_limit_) + , log(&Poco::Logger::get(fmt::format("StorageKeeperMap ({})", table_id.getNameForLogs()))) +{ + std::string path_prefix = context_->getConfigRef().getString("keeper_map_path_prefix", ""); + if (path_prefix.empty()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "KeeperMap is disabled because 'keeper_map_path_prefix' config is not defined"); + + verifyTableId(table_id); + + setInMemoryMetadata(metadata); + + WriteBufferFromOwnString out; + out << "KeeperMap metadata format version: 1\n" + << "columns: " << metadata.columns.toString() + << "primary key: " << formattedAST(metadata.getPrimaryKey().expression_list_ast) << "\n"; + metadata_string = out.str(); + + if (root_path.empty()) + throw Exception("root_path should not be empty", ErrorCodes::BAD_ARGUMENTS); + if (!root_path.starts_with('/')) + throw Exception("root_path should start with '/'", ErrorCodes::BAD_ARGUMENTS); + + auto config_keys_limit = context_->getConfigRef().getUInt64("keeper_map_keys_limit", 0); + if (config_keys_limit != 0 && (keys_limit == 0 || keys_limit > config_keys_limit)) + { + LOG_WARNING( + log, + "Keys limit defined by argument ({}) is larger than the one defined by 'keeper_map_keys_limit' config ({}). Will use " + "config defined value", + keys_limit, + config_keys_limit); + keys_limit = config_keys_limit; + } + else if (keys_limit > 0) + { + LOG_INFO(log, "Keys limit will be set to {}", keys_limit); + } + + auto root_path_fs = fs::path(path_prefix) / std::string_view{root_path}.substr(1); + root_path = root_path_fs.generic_string(); + + data_path = root_path_fs / "data"; + + auto metadata_path_fs = root_path_fs / "metadata"; + metadata_path = metadata_path_fs; + tables_path = metadata_path_fs / "tables"; + + auto table_unique_id = toString(table_id.uuid) + toString(ServerUUID::get()); + table_path = fs::path(tables_path) / table_unique_id; + + dropped_path = metadata_path_fs / "dropped"; + dropped_lock_path = fs::path(dropped_path) / "lock"; + + if (attach) + { + checkTable(); + return; + } + + auto client = getClient(); + + if (root_path != "/" && !client->exists(root_path)) + { + LOG_TRACE(log, "Creating root path {}", root_path); + client->createAncestors(root_path); + client->createIfNotExists(root_path, ""); + } + + for (size_t i = 0; i < 1000; ++i) + { + if (client->exists(dropped_path)) + { + LOG_INFO(log, "Removing leftover nodes"); + auto code = client->tryCreate(dropped_lock_path, "", zkutil::CreateMode::Ephemeral); + + if (code == Coordination::Error::ZNONODE) + { + LOG_INFO(log, "Someone else removed leftover nodes"); + } + else if (code == Coordination::Error::ZNODEEXISTS) + { + LOG_INFO(log, "Someone else is removing leftover nodes"); + continue; + } + else if (code != Coordination::Error::ZOK) + { + throw Coordination::Exception(code, dropped_lock_path); + } + else + { + auto metadata_drop_lock = zkutil::EphemeralNodeHolder::existing(dropped_lock_path, *client); + if (!dropTable(client, metadata_drop_lock)) + continue; + } + } + + std::string stored_metadata_string; + auto exists = client->tryGet(metadata_path, stored_metadata_string); + + if (exists) + { + // this requires same name for columns + // maybe we can do a smarter comparison for columns and primary key expression + if (stored_metadata_string != metadata_string) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Path {} is already used but the stored table definition doesn't match. Stored metadata: {}", + root_path, + stored_metadata_string); + } + else + { + auto code = client->tryCreate(metadata_path, metadata_string, zkutil::CreateMode::Persistent); + if (code == Coordination::Error::ZNODEEXISTS) + continue; + else if (code != Coordination::Error::ZOK) + throw Coordination::Exception(code, metadata_path); + } + + client->createIfNotExists(tables_path, ""); + + auto code = client->tryCreate(table_path, "", zkutil::CreateMode::Persistent); + + if (code == Coordination::Error::ZOK) + { + // metadata now should be guaranteed to exist because we added our UUID to the tables_path + client->createIfNotExists(data_path, ""); + table_is_valid = true; + return; + } + + if (code == Coordination::Error::ZNONODE) + LOG_INFO(log, "Metadata nodes were deleted in background, will retry"); + else + throw Coordination::Exception(code, table_path); + } + + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cannot create metadata for table, because it is removed concurrently or because of wrong root_path ({})", root_path); +} + + +Pipe StorageKeeperMap::read( + const Names & column_names, + const StorageSnapshotPtr & storage_snapshot, + SelectQueryInfo & query_info, + ContextPtr context_, + QueryProcessingStage::Enum /*processed_stage*/, + size_t max_block_size, + unsigned num_streams) +{ + checkTable(); + storage_snapshot->check(column_names); + + FieldVectorPtr filtered_keys; + bool all_scan; + + Block sample_block = storage_snapshot->metadata->getSampleBlock(); + auto primary_key_type = sample_block.getByName(primary_key).type; + std::tie(filtered_keys, all_scan) = getFilterKeys(primary_key, primary_key_type, query_info, context_); + + const auto process_keys = [&](KeyContainerPtr keys) -> Pipe + { + if (keys->empty()) + return {}; + + ::sort(keys->begin(), keys->end()); + keys->erase(std::unique(keys->begin(), keys->end()), keys->end()); + + Pipes pipes; + + size_t num_keys = keys->size(); + size_t num_threads = std::min(num_streams, keys->size()); + + assert(num_keys <= std::numeric_limits::max()); + assert(num_threads <= std::numeric_limits::max()); + + for (size_t thread_idx = 0; thread_idx < num_threads; ++thread_idx) + { + size_t begin = num_keys * thread_idx / num_threads; + size_t end = num_keys * (thread_idx + 1) / num_threads; + + using KeyContainer = typename KeyContainerPtr::element_type; + pipes.emplace_back(std::make_shared>( + *this, sample_block, max_block_size, keys, keys->begin() + begin, keys->begin() + end)); + } + return Pipe::unitePipes(std::move(pipes)); + }; + + auto client = getClient(); + if (all_scan) + return process_keys(std::make_shared>(client->getChildren(data_path))); + + return process_keys(std::move(filtered_keys)); +} + +SinkToStoragePtr StorageKeeperMap::write(const ASTPtr & /*query*/, const StorageMetadataPtr & metadata_snapshot, ContextPtr /*context*/) +{ + checkTable(); + return std::make_shared(*this, metadata_snapshot); +} + +void StorageKeeperMap::truncate(const ASTPtr &, const StorageMetadataPtr &, ContextPtr, TableExclusiveLockHolder &) +{ + checkTable(); + auto client = getClient(); + client->tryRemoveChildrenRecursive(data_path, true); +} + +bool StorageKeeperMap::dropTable(zkutil::ZooKeeperPtr zookeeper, const zkutil::EphemeralNodeHolder::Ptr & metadata_drop_lock) +{ + zookeeper->removeChildrenRecursive(data_path); + + bool completely_removed = false; + Coordination::Requests ops; + ops.emplace_back(zkutil::makeRemoveRequest(metadata_drop_lock->getPath(), -1)); + ops.emplace_back(zkutil::makeRemoveRequest(dropped_path, -1)); + ops.emplace_back(zkutil::makeRemoveRequest(data_path, -1)); + ops.emplace_back(zkutil::makeRemoveRequest(metadata_path, -1)); + + Coordination::Responses responses; + auto code = zookeeper->tryMulti(ops, responses); + using enum Coordination::Error; + switch (code) + { + case ZOK: + { + metadata_drop_lock->setAlreadyRemoved(); + completely_removed = true; + LOG_INFO(log, "Metadata ({}) and data ({}) was successfully removed from ZooKeeper", metadata_path, data_path); + break; + } + case ZNONODE: + throw Exception(ErrorCodes::LOGICAL_ERROR, "There is a race condition between creation and removal of metadata. It's a bug"); + case ZNOTEMPTY: + LOG_ERROR(log, "Metadata was not completely removed from ZooKeeper"); + break; + default: + zkutil::KeeperMultiException::check(code, ops, responses); + break; + } + return completely_removed; +} + +void StorageKeeperMap::drop() +{ + checkTable(); + auto client = getClient(); + + client->remove(table_path); + + if (!client->getChildren(tables_path).empty()) + return; + + Coordination::Requests ops; + Coordination::Responses responses; + + ops.emplace_back(zkutil::makeRemoveRequest(tables_path, -1)); + ops.emplace_back(zkutil::makeCreateRequest(dropped_path, "", zkutil::CreateMode::Persistent)); + ops.emplace_back(zkutil::makeCreateRequest(dropped_lock_path, "", zkutil::CreateMode::Ephemeral)); + + auto code = client->tryMulti(ops, responses); + + if (code == Coordination::Error::ZNONODE || code == Coordination::Error::ZNODEEXISTS) + { + LOG_INFO(log, "Metadata is being removed by another table"); + return; + } + else if (code == Coordination::Error::ZNOTEMPTY) + { + LOG_WARNING(log, "Another table is using the same path, metadata will not be deleted"); + return; + } + else if (code != Coordination::Error::ZOK) + zkutil::KeeperMultiException::check(code, ops, responses); + + auto metadata_drop_lock = zkutil::EphemeralNodeHolder::existing(dropped_lock_path, *client); + dropTable(client, metadata_drop_lock); +} + +zkutil::ZooKeeperPtr StorageKeeperMap::getClient() const +{ + std::lock_guard lock{zookeeper_mutex}; + if (!zookeeper_client || zookeeper_client->expired()) + { + zookeeper_client = nullptr; + if (zookeeper_name == "default") + zookeeper_client = getContext()->getZooKeeper(); + else + zookeeper_client = getContext()->getAuxiliaryZooKeeper(zookeeper_name); + + zookeeper_client->sync(root_path); + } + + return zookeeper_client; +} + +const std::string & StorageKeeperMap::dataPath() const +{ + return data_path; +} + +std::string StorageKeeperMap::fullPathForKey(const std::string_view key) const +{ + return fs::path(data_path) / key; +} + +UInt64 StorageKeeperMap::keysLimit() const +{ + return keys_limit; +} + +std::optional StorageKeeperMap::isTableValid() const +{ + std::lock_guard lock{init_mutex}; + if (table_is_valid.has_value()) + return *table_is_valid; + + [&] + { + try + { + auto client = getClient(); + + std::string stored_metadata_string; + Coordination::Stat metadata_stat; + client->tryGet(metadata_path, stored_metadata_string, &metadata_stat); + + if (metadata_stat.numChildren == 0) + { + table_is_valid = false; + return; + } + + if (metadata_string != stored_metadata_string) + { + LOG_ERROR( + log, + "Table definition does not match to the one stored in the path {}. Stored definition: {}", + root_path, + stored_metadata_string); + table_is_valid = false; + return; + } + + // validate all metadata and data nodes are present + Coordination::Requests requests; + requests.push_back(zkutil::makeCheckRequest(table_path, -1)); + requests.push_back(zkutil::makeCheckRequest(data_path, -1)); + requests.push_back(zkutil::makeCheckRequest(dropped_path, -1)); + + Coordination::Responses responses; + client->tryMulti(requests, responses); + + table_is_valid = false; + if (responses[0]->error != Coordination::Error::ZOK) + { + LOG_ERROR(log, "Table node ({}) is missing", table_path); + return; + } + + if (responses[1]->error != Coordination::Error::ZOK) + { + LOG_ERROR(log, "Data node ({}) is missing", data_path); + return; + } + + if (responses[2]->error == Coordination::Error::ZOK) + { + LOG_ERROR(log, "Tables with root node {} are being dropped", root_path); + return; + } + + table_is_valid = true; + } + catch (const Coordination::Exception & e) + { + tryLogCurrentException(log); + + if (!Coordination::isHardwareError(e.code)) + table_is_valid = false; + } + }(); + + return table_is_valid; +} + +Chunk StorageKeeperMap::getByKeys(const ColumnsWithTypeAndName & keys, PaddedPODArray & null_map, const Names &) const +{ + if (keys.size() != 1) + throw Exception(ErrorCodes::LOGICAL_ERROR, "StorageKeeperMap supports only one key, got: {}", keys.size()); + + auto raw_keys = serializeKeysToRawString(keys[0]); + + if (raw_keys.size() != keys[0].column->size()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Assertion failed: {} != {}", raw_keys.size(), keys[0].column->size()); + + return getBySerializedKeys(raw_keys, &null_map); +} + +Chunk StorageKeeperMap::getBySerializedKeys(const std::span keys, PaddedPODArray * null_map) const +{ + Block sample_block = getInMemoryMetadataPtr()->getSampleBlock(); + MutableColumns columns = sample_block.cloneEmptyColumns(); + size_t primary_key_pos = getPrimaryKeyPos(sample_block, getPrimaryKey()); + + if (null_map) + { + null_map->clear(); + null_map->resize_fill(keys.size(), 1); + } + + auto client = getClient(); + + std::vector> values; + values.reserve(keys.size()); + + for (const auto & key : keys) + { + const auto full_path = fullPathForKey(key); + values.emplace_back(client->asyncTryGet(full_path)); + } + + auto wait_until = std::chrono::system_clock::now() + std::chrono::milliseconds(Coordination::DEFAULT_OPERATION_TIMEOUT_MS); + + for (size_t i = 0; i < keys.size(); ++i) + { + auto & value = values[i]; + if (value.wait_until(wait_until) != std::future_status::ready) + throw DB::Exception(ErrorCodes::KEEPER_EXCEPTION, "Failed to fetch values: timeout"); + + auto response = value.get(); + Coordination::Error code = response.error; + + if (code == Coordination::Error::ZOK) + { + fillColumns(base64Decode(keys[i], true), response.data, primary_key_pos, sample_block, columns); + } + else if (code == Coordination::Error::ZNONODE) + { + if (null_map) + { + (*null_map)[i] = 0; + for (size_t col_idx = 0; col_idx < sample_block.columns(); ++col_idx) + columns[col_idx]->insert(sample_block.getByPosition(col_idx).type->getDefault()); + } + } + else + { + throw DB::Exception(ErrorCodes::KEEPER_EXCEPTION, "Failed to fetch value: {}", code); + } + } + + size_t num_rows = columns.at(0)->size(); + return Chunk(std::move(columns), num_rows); +} + +Block StorageKeeperMap::getSampleBlock(const Names &) const +{ + auto metadata = getInMemoryMetadataPtr(); + return metadata->getSampleBlock(); +} + +void StorageKeeperMap::checkTableCanBeRenamed(const StorageID & new_name) const +{ + verifyTableId(new_name); +} + +void StorageKeeperMap::rename(const String & /*new_path_to_table_data*/, const StorageID & new_table_id) +{ + checkTableCanBeRenamed(new_table_id); + renameInMemory(new_table_id); +} + +namespace +{ + +StoragePtr create(const StorageFactory::Arguments & args) +{ + ASTs & engine_args = args.engine_args; + if (engine_args.empty() || engine_args.size() > 2) + throw Exception( + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, + "Storage KeeperMap requires 1-3 arguments:\n" + "root_path: path in the Keeper where the values will be stored (required)\n" + "keys_limit: number of keys allowed to be stored, 0 is no limit (default: 0)"); + + const auto root_path_node = evaluateConstantExpressionAsLiteral(engine_args[0], args.getLocalContext()); + auto root_path = checkAndGetLiteralArgument(root_path_node, "root_path"); + + UInt64 keys_limit = 0; + if (engine_args.size() > 1) + keys_limit = checkAndGetLiteralArgument(engine_args[1], "keys_limit"); + + StorageInMemoryMetadata metadata; + metadata.setColumns(args.columns); + metadata.setConstraints(args.constraints); + + if (!args.storage_def->primary_key) + throw Exception("StorageKeeperMap requires one column in primary key", ErrorCodes::BAD_ARGUMENTS); + + metadata.primary_key = KeyDescription::getKeyFromAST(args.storage_def->primary_key->ptr(), metadata.columns, args.getContext()); + auto primary_key_names = metadata.getColumnsRequiredForPrimaryKey(); + if (primary_key_names.size() != 1) + throw Exception("StorageKeeperMap requires one column in primary key", ErrorCodes::BAD_ARGUMENTS); + + return std::make_shared( + args.getContext(), args.table_id, metadata, args.query.attach, primary_key_names[0], root_path, keys_limit); +} + +} + +void registerStorageKeeperMap(StorageFactory & factory) +{ + factory.registerStorage( + "KeeperMap", + create, + { + .supports_sort_order = true, + .supports_parallel_insert = true, + }); +} + +} diff --git a/src/Storages/StorageKeeperMap.h b/src/Storages/StorageKeeperMap.h new file mode 100644 index 00000000000..87861362e42 --- /dev/null +++ b/src/Storages/StorageKeeperMap.h @@ -0,0 +1,138 @@ +#pragma once + +#include +#include + +#include +#include +#include +#include +#include + +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int INVALID_STATE; +} + +// KV store using (Zoo|CH)Keeper +class StorageKeeperMap final : public IStorage, public IKeyValueEntity, WithContext +{ +public: + StorageKeeperMap( + ContextPtr context_, + const StorageID & table_id, + const StorageInMemoryMetadata & metadata, + bool attach, + std::string_view primary_key_, + const std::string & root_path_, + UInt64 keys_limit_); + + Pipe read( + const Names & column_names, + const StorageSnapshotPtr & storage_snapshot, + SelectQueryInfo & query_info, + ContextPtr context, + QueryProcessingStage::Enum processed_stage, + size_t max_block_size, + unsigned num_streams) override; + + SinkToStoragePtr write(const ASTPtr & query, const StorageMetadataPtr & metadata_snapshot, ContextPtr context) override; + + void truncate(const ASTPtr &, const StorageMetadataPtr &, ContextPtr, TableExclusiveLockHolder &) override; + void drop() override; + + std::string getName() const override { return "KeeperMap"; } + Names getPrimaryKey() const override { return {primary_key}; } + + Chunk getByKeys(const ColumnsWithTypeAndName & keys, PaddedPODArray & null_map, const Names &) const override; + Chunk getBySerializedKeys(std::span keys, PaddedPODArray * null_map) const; + + Block getSampleBlock(const Names &) const override; + + void checkTableCanBeRenamed(const StorageID & new_name) const override; + void rename(const String & new_path_to_table_data, const StorageID & new_table_id) override; + + bool supportsParallelInsert() const override { return true; } + bool supportsIndexForIn() const override { return true; } + bool mayBenefitFromIndexForIn( + const ASTPtr & node, ContextPtr /*query_context*/, const StorageMetadataPtr & /*metadata_snapshot*/) const override + { + return node->getColumnName() == primary_key; + } + + zkutil::ZooKeeperPtr getClient() const; + const std::string & dataPath() const; + std::string fullPathForKey(std::string_view key) const; + + UInt64 keysLimit() const; + + template + void checkTable() const + { + auto is_table_valid = isTableValid(); + if (!is_table_valid.has_value()) + { + static constexpr std::string_view error_msg = "Failed to activate table because of connection issues. It will be activated " + "once a connection is established and metadata is verified"; + if constexpr (throw_on_error) + throw Exception(ErrorCodes::INVALID_STATE, error_msg); + else + { + LOG_ERROR(log, fmt::runtime(error_msg)); + return; + } + } + + if (!*is_table_valid) + { + static constexpr std::string_view error_msg + = "Failed to activate table because of invalid metadata in ZooKeeper. Please DETACH table"; + if constexpr (throw_on_error) + throw Exception(ErrorCodes::INVALID_STATE, error_msg); + else + { + LOG_ERROR(log, fmt::runtime(error_msg)); + return; + } + } + } + +private: + bool dropTable(zkutil::ZooKeeperPtr zookeeper, const zkutil::EphemeralNodeHolder::Ptr & metadata_drop_lock); + + std::optional isTableValid() const; + + std::string root_path; + std::string primary_key; + + std::string data_path; + + std::string metadata_path; + + std::string tables_path; + std::string table_path; + + std::string dropped_path; + std::string dropped_lock_path; + + std::string zookeeper_name; + + std::string metadata_string; + + uint64_t keys_limit{0}; + + mutable std::mutex zookeeper_mutex; + mutable zkutil::ZooKeeperPtr zookeeper_client{nullptr}; + + mutable std::mutex init_mutex; + mutable std::optional table_is_valid; + + Poco::Logger * log; +}; + +} diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index 5adc1974257..e4062734352 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -335,6 +335,13 @@ void StorageMergeTree::alter( mutation_version = startMutation(maybe_mutation_commands, local_context); } + { + /// Reset Object columns, because column of type + /// Object may be added or dropped by alter. + auto parts_lock = lockParts(); + resetObjectColumnsFromActiveParts(parts_lock); + } + /// Always execute required mutations synchronously, because alters /// should be executed in sequential order. if (!maybe_mutation_commands.empty()) diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index a2d10e57f8f..295f201191a 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -3649,7 +3649,7 @@ void StorageReplicatedMergeTree::updateQuorum(const String & part_name, bool is_ if (quorum_entry.replicas.size() >= quorum_entry.required_number_of_replicas) { /// The quorum is reached. Delete the node, and update information about the last part that was successfully written with quorum. - LOG_TRACE(log, "Got {} (of {}) replicas confirmed quorum {}, going to remove node", + LOG_TRACE(log, "Got {} (of {} required) replicas confirmed quorum {}, going to remove node", quorum_entry.replicas.size(), quorum_entry.required_number_of_replicas, quorum_status_path); Coordination::Requests ops; @@ -4649,6 +4649,13 @@ bool StorageReplicatedMergeTree::executeMetadataAlter(const StorageReplicatedMer LOG_INFO(log, "Applied changes to the metadata of the table. Current metadata version: {}", metadata_version); } + { + /// Reset Object columns, because column of type + /// Object may be added or dropped by alter. + auto parts_lock = lockParts(); + resetObjectColumnsFromActiveParts(parts_lock); + } + /// This transaction may not happen, but it's OK, because on the next retry we will eventually create/update this node /// TODO Maybe do in in one transaction for Replicated database? zookeeper->createOrUpdate(fs::path(replica_path) / "metadata_version", std::to_string(metadata_version), zkutil::CreateMode::Persistent); @@ -7597,7 +7604,7 @@ std::pair StorageReplicatedMergeTree::unlockSharedData(const IMer } else { - LOG_TRACE(log, "Part {} looks temporary, because checksums file doesn't exists, blobs can be removed", part.name); + LOG_TRACE(log, "Part {} looks temporary, because {} file doesn't exists, blobs can be removed", part.name, IMergeTreeDataPart::FILE_FOR_REFERENCES_CHECK); /// Temporary part with some absent file cannot be locked in shared mode return std::make_pair(true, NameSet{}); } diff --git a/src/Storages/System/StorageSystemDDLWorkerQueue.cpp b/src/Storages/System/StorageSystemDDLWorkerQueue.cpp index 111ea343398..67867b6c577 100644 --- a/src/Storages/System/StorageSystemDDLWorkerQueue.cpp +++ b/src/Storages/System/StorageSystemDDLWorkerQueue.cpp @@ -205,9 +205,9 @@ static void fillStatusColumns(MutableColumns & res_columns, size_t & col, void StorageSystemDDLWorkerQueue::fillData(MutableColumns & res_columns, ContextPtr context, const SelectQueryInfo &) const { - zkutil::ZooKeeperPtr zookeeper = context->getZooKeeper(); - fs::path ddl_zookeeper_path = context->getConfigRef().getString("distributed_ddl.path", "/clickhouse/task_queue/ddl/"); - + auto& ddl_worker = context->getDDLWorker(); + fs::path ddl_zookeeper_path = ddl_worker.getQueueDir(); + zkutil::ZooKeeperPtr zookeeper = ddl_worker.getAndSetZooKeeper(); Strings ddl_task_paths = zookeeper->getChildren(ddl_zookeeper_path); GetResponseFutures ddl_task_futures; diff --git a/src/Storages/System/StorageSystemModels.cpp b/src/Storages/System/StorageSystemModels.cpp index 4a4dbbc69df..d06f97a3f54 100644 --- a/src/Storages/System/StorageSystemModels.cpp +++ b/src/Storages/System/StorageSystemModels.cpp @@ -1,11 +1,11 @@ #include +#include #include #include #include #include #include -#include -#include +#include namespace DB @@ -14,45 +14,24 @@ namespace DB NamesAndTypesList StorageSystemModels::getNamesAndTypes() { return { - { "name", std::make_shared() }, - { "status", std::make_shared(getStatusEnumAllPossibleValues()) }, - { "origin", std::make_shared() }, + { "model_path", std::make_shared() }, { "type", std::make_shared() }, { "loading_start_time", std::make_shared() }, { "loading_duration", std::make_shared() }, - //{ "creation_time", std::make_shared() }, - { "last_exception", std::make_shared() }, }; } void StorageSystemModels::fillData(MutableColumns & res_columns, ContextPtr context, const SelectQueryInfo &) const { - const auto & external_models_loader = context->getExternalModelsLoader(); - auto load_results = external_models_loader.getLoadResults(); + auto bridge_helper = std::make_unique(context); + ExternalModelInfos infos = bridge_helper->listModels(); - for (const auto & load_result : load_results) + for (const auto & info : infos) { - res_columns[0]->insert(load_result.name); - res_columns[1]->insert(static_cast(load_result.status)); - res_columns[2]->insert(load_result.config ? load_result.config->path : ""); - - if (load_result.object) - { - const auto model_ptr = std::static_pointer_cast(load_result.object); - res_columns[3]->insert(model_ptr->getTypeName()); - } - else - { - res_columns[3]->insertDefault(); - } - - res_columns[4]->insert(static_cast(std::chrono::system_clock::to_time_t(load_result.loading_start_time))); - res_columns[5]->insert(std::chrono::duration_cast>(load_result.loading_duration).count()); - - if (load_result.exception) - res_columns[6]->insert(getExceptionMessage(load_result.exception, false)); - else - res_columns[6]->insertDefault(); + res_columns[0]->insert(info.model_path); + res_columns[1]->insert(info.model_type); + res_columns[2]->insert(static_cast(std::chrono::system_clock::to_time_t(info.loading_start_time))); + res_columns[3]->insert(std::chrono::duration_cast>(info.loading_duration).count()); } } diff --git a/src/Storages/registerStorages.cpp b/src/Storages/registerStorages.cpp index 575b3de7ae2..055270be4ae 100644 --- a/src/Storages/registerStorages.cpp +++ b/src/Storages/registerStorages.cpp @@ -88,6 +88,7 @@ void registerStorageFileLog(StorageFactory & factory); void registerStorageSQLite(StorageFactory & factory); #endif +void registerStorageKeeperMap(StorageFactory & factory); void registerStorages() { @@ -171,6 +172,8 @@ void registerStorages() #if USE_SQLITE registerStorageSQLite(factory); #endif + + registerStorageKeeperMap(factory); } } diff --git a/src/Storages/tests/gtest_storage_log.cpp b/src/Storages/tests/gtest_storage_log.cpp index 3fa2f93b484..f1079a9ee10 100644 --- a/src/Storages/tests/gtest_storage_log.cpp +++ b/src/Storages/tests/gtest_storage_log.cpp @@ -5,8 +5,6 @@ #include #include #include -#include -#include #include #include #include @@ -18,18 +16,12 @@ #include #include #include -#include #include #include #include #include #include -#if !defined(__clang__) -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wsuggest-override" -#endif - DB::StoragePtr createStorage(DB::DiskPtr & disk) { diff --git a/tests/ci/build_check.py b/tests/ci/build_check.py index f58c7a74dfe..d668dbe0498 100644 --- a/tests/ci/build_check.py +++ b/tests/ci/build_check.py @@ -291,7 +291,9 @@ def main(): logging.info("Will try to fetch cache for our build") try: - get_ccache_if_not_exists(ccache_path, s3_helper, pr_info.number, TEMP_PATH) + get_ccache_if_not_exists( + ccache_path, s3_helper, pr_info.number, TEMP_PATH, pr_info.release_pr + ) except Exception as e: # In case there are issues with ccache, remove the path and do not fail a build logging.info("Failed to get ccache, building without it. Error: %s", e) diff --git a/tests/ci/ccache_utils.py b/tests/ci/ccache_utils.py index cfe07363589..864b3a8f9b6 100644 --- a/tests/ci/ccache_utils.py +++ b/tests/ci/ccache_utils.py @@ -11,6 +11,7 @@ import requests # type: ignore from compress_files import decompress_fast, compress_fast from env_helper import S3_DOWNLOAD, S3_BUILDS_BUCKET +from s3_helper import S3Helper DOWNLOAD_RETRIES_COUNT = 5 @@ -57,12 +58,19 @@ def dowload_file_with_progress(url, path): def get_ccache_if_not_exists( - path_to_ccache_dir, s3_helper, current_pr_number, temp_path + path_to_ccache_dir: str, + s3_helper: S3Helper, + current_pr_number: int, + temp_path: str, + release_pr: int, ) -> int: """returns: number of PR for downloaded PR. -1 if ccache not found""" ccache_name = os.path.basename(path_to_ccache_dir) cache_found = False prs_to_check = [current_pr_number] + # Release PR is either 0 or defined + if release_pr: + prs_to_check.append(release_pr) ccache_pr = -1 if current_pr_number != 0: prs_to_check.append(0) diff --git a/tests/ci/cherry_pick.py b/tests/ci/cherry_pick.py index 064a0b3add1..d1c9d3d394c 100644 --- a/tests/ci/cherry_pick.py +++ b/tests/ci/cherry_pick.py @@ -44,11 +44,11 @@ from ssh import SSHKey class Labels: - LABEL_MUST_BACKPORT = "pr-must-backport" - LABEL_BACKPORT = "pr-backport" - LABEL_BACKPORTED = "pr-backported" - LABEL_CHERRYPICK = "pr-cherrypick" - LABEL_DO_NOT_TEST = "do not test" + MUST_BACKPORT = "pr-must-backport" + BACKPORT = "pr-backport" + BACKPORTS_CREATED = "pr-backports-created" + CHERRYPICK = "pr-cherrypick" + DO_NOT_TEST = "do not test" class ReleaseBranch: @@ -204,8 +204,8 @@ Merge it only if you intend to backport changes to the target branch, otherwise base=self.backport_branch, head=self.cherrypick_branch, ) - self.cherrypick_pr.add_to_labels(Labels.LABEL_CHERRYPICK) - self.cherrypick_pr.add_to_labels(Labels.LABEL_DO_NOT_TEST) + self.cherrypick_pr.add_to_labels(Labels.CHERRYPICK) + self.cherrypick_pr.add_to_labels(Labels.DO_NOT_TEST) self._assign_new_pr(self.cherrypick_pr) def create_backport(self): @@ -236,7 +236,7 @@ Merge it only if you intend to backport changes to the target branch, otherwise base=self.name, head=self.backport_branch, ) - self.backport_pr.add_to_labels(Labels.LABEL_BACKPORT) + self.backport_pr.add_to_labels(Labels.BACKPORT) self._assign_new_pr(self.backport_pr) def _assign_new_pr(self, new_pr: PullRequest): @@ -321,8 +321,8 @@ class Backport: tomorrow = date.today() + timedelta(days=1) logging.info("Receive PRs suppose to be backported") self.prs_for_backport = self.gh.get_pulls_from_search( - query=f"{self._query} -label:pr-backported", - label=",".join(self.labels_to_backport + [Labels.LABEL_MUST_BACKPORT]), + query=f"{self._query} -label:{Labels.BACKPORTS_CREATED}", + label=",".join(self.labels_to_backport + [Labels.MUST_BACKPORT]), merged=[since_date, tomorrow], ) logging.info( @@ -342,7 +342,7 @@ class Backport: def process_pr(self, pr: PullRequest): pr_labels = [label.name for label in pr.labels] - if Labels.LABEL_MUST_BACKPORT in pr_labels: + if Labels.MUST_BACKPORT in pr_labels: branches = [ ReleaseBranch(br, pr) for br in self.release_branches ] # type: List[ReleaseBranch] @@ -407,11 +407,11 @@ class Backport: if self.dry_run: logging.info("DRY RUN: would mark PR #%s as done", pr.number) return - pr.add_to_labels(Labels.LABEL_BACKPORTED) + pr.add_to_labels(Labels.BACKPORTS_CREATED) logging.info( "PR #%s is successfully labeled with `%s`", pr.number, - Labels.LABEL_BACKPORTED, + Labels.BACKPORTS_CREATED, ) @property diff --git a/tests/ci/ci_config.py b/tests/ci/ci_config.py index fa68d1982d2..a31f2298a58 100644 --- a/tests/ci/ci_config.py +++ b/tests/ci/ci_config.py @@ -8,7 +8,7 @@ BuildConfig = Dict[str, ConfValue] CI_CONFIG = { "build_config": { "package_release": { - "compiler": "clang-14", + "compiler": "clang-15", "build_type": "", "sanitizer": "", "package_type": "deb", @@ -19,7 +19,7 @@ CI_CONFIG = { "with_coverage": False, }, "coverity": { - "compiler": "clang-14", + "compiler": "clang-15", "build_type": "", "sanitizer": "", "package_type": "coverity", @@ -29,7 +29,7 @@ CI_CONFIG = { "official": False, }, "package_aarch64": { - "compiler": "clang-14-aarch64", + "compiler": "clang-15-aarch64", "build_type": "", "sanitizer": "", "package_type": "deb", @@ -40,7 +40,7 @@ CI_CONFIG = { "with_coverage": False, }, "package_asan": { - "compiler": "clang-14", + "compiler": "clang-15", "build_type": "", "sanitizer": "address", "package_type": "deb", @@ -49,7 +49,7 @@ CI_CONFIG = { "with_coverage": False, }, "package_ubsan": { - "compiler": "clang-14", + "compiler": "clang-15", "build_type": "", "sanitizer": "undefined", "package_type": "deb", @@ -67,7 +67,7 @@ CI_CONFIG = { "with_coverage": False, }, "package_msan": { - "compiler": "clang-14", + "compiler": "clang-15", "build_type": "", "sanitizer": "memory", "package_type": "deb", @@ -76,7 +76,7 @@ CI_CONFIG = { "with_coverage": False, }, "package_debug": { - "compiler": "clang-14", + "compiler": "clang-15", "build_type": "debug", "sanitizer": "", "package_type": "deb", @@ -85,7 +85,7 @@ CI_CONFIG = { "with_coverage": False, }, "binary_release": { - "compiler": "clang-14", + "compiler": "clang-15", "build_type": "", "sanitizer": "", "package_type": "binary", @@ -94,7 +94,7 @@ CI_CONFIG = { "with_coverage": False, }, "binary_tidy": { - "compiler": "clang-14", + "compiler": "clang-15", "build_type": "debug", "sanitizer": "", "package_type": "binary", @@ -104,7 +104,7 @@ CI_CONFIG = { "with_coverage": False, }, "binary_shared": { - "compiler": "clang-14", + "compiler": "clang-15", "build_type": "", "sanitizer": "", "package_type": "binary", @@ -113,7 +113,7 @@ CI_CONFIG = { "with_coverage": False, }, "binary_darwin": { - "compiler": "clang-14-darwin", + "compiler": "clang-15-darwin", "build_type": "", "sanitizer": "", "package_type": "binary", @@ -123,7 +123,7 @@ CI_CONFIG = { "with_coverage": False, }, "binary_aarch64": { - "compiler": "clang-14-aarch64", + "compiler": "clang-15-aarch64", "build_type": "", "sanitizer": "", "package_type": "binary", @@ -132,7 +132,7 @@ CI_CONFIG = { "with_coverage": False, }, "binary_freebsd": { - "compiler": "clang-14-freebsd", + "compiler": "clang-15-freebsd", "build_type": "", "sanitizer": "", "package_type": "binary", @@ -142,7 +142,7 @@ CI_CONFIG = { "with_coverage": False, }, "binary_darwin_aarch64": { - "compiler": "clang-14-darwin-aarch64", + "compiler": "clang-15-darwin-aarch64", "build_type": "", "sanitizer": "", "package_type": "binary", @@ -152,7 +152,7 @@ CI_CONFIG = { "with_coverage": False, }, "binary_ppc64le": { - "compiler": "clang-14-ppc64le", + "compiler": "clang-15-ppc64le", "build_type": "", "sanitizer": "", "package_type": "binary", @@ -162,7 +162,7 @@ CI_CONFIG = { "with_coverage": False, }, "binary_amd64sse2": { - "compiler": "clang-14-amd64sse2", + "compiler": "clang-15-amd64sse2", "build_type": "", "sanitizer": "", "package_type": "binary", @@ -342,7 +342,7 @@ CI_CONFIG = { }, "Performance Comparison Aarch64": { "required_build": "package_aarch64", - "test_grep_exclude_filter": "constant_column_search", + "test_grep_exclude_filter": "", }, }, } # type: dict diff --git a/tests/ci/fast_test_check.py b/tests/ci/fast_test_check.py index 038289406de..03e42726808 100644 --- a/tests/ci/fast_test_check.py +++ b/tests/ci/fast_test_check.py @@ -125,7 +125,7 @@ if __name__ == "__main__": logging.info("Will try to fetch cache for our build") ccache_for_pr = get_ccache_if_not_exists( - cache_path, s3_helper, pr_info.number, temp_path + cache_path, s3_helper, pr_info.number, temp_path, pr_info.release_pr ) upload_master_ccache = ccache_for_pr in (-1, 0) diff --git a/tests/ci/pr_info.py b/tests/ci/pr_info.py index 2acd0e4c811..77421ddac32 100644 --- a/tests/ci/pr_info.py +++ b/tests/ci/pr_info.py @@ -86,7 +86,7 @@ class PRInfo: self.changed_files = set() # type: Set[str] self.body = "" self.diff_urls = [] - self.release_pr = "" + self.release_pr = 0 ref = github_event.get("ref", "refs/head/master") if ref and ref.startswith("refs/heads/"): ref = ref[11:] diff --git a/tests/ci/worker/init_runner.sh b/tests/ci/worker/init_runner.sh index 3dfd1761e88..66a38a6a37d 100644 --- a/tests/ci/worker/init_runner.sh +++ b/tests/ci/worker/init_runner.sh @@ -76,13 +76,13 @@ if [[ ${ROOT_STAT[0]} -lt 3000000 ]] || [[ ${ROOT_STAT[1]} -lt 5 ]]; then fi # shellcheck disable=SC2046 -docker kill $(docker ps -q) ||: +docker ps --quiet | xargs --no-run-if-empty docker kill ||: # shellcheck disable=SC2046 -docker rm -f $(docker ps -a -q) ||: +docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # If we have hanged containers after the previous commands, than we have a hanged one # and should restart the daemon -if [ "$(docker ps -a -q)" ]; then +if [ "$(docker ps --all --quiet)" ]; then # Systemd service of docker has StartLimitBurst=3 and StartLimitInterval=60s, # that's why we try restarting it for long for i in {1..25}; diff --git a/tests/config/config.d/enable_keeper_map.xml b/tests/config/config.d/enable_keeper_map.xml new file mode 100644 index 00000000000..b4cbb6a954b --- /dev/null +++ b/tests/config/config.d/enable_keeper_map.xml @@ -0,0 +1,3 @@ + + /test_keeper_map + diff --git a/tests/config/install.sh b/tests/config/install.sh index e4706b53e4e..d4c71212423 100755 --- a/tests/config/install.sh +++ b/tests/config/install.sh @@ -50,6 +50,7 @@ ln -sf $SRC_PATH/config.d/session_log.xml $DEST_SERVER_PATH/config.d/ ln -sf $SRC_PATH/config.d/system_unfreeze.xml $DEST_SERVER_PATH/config.d/ ln -sf $SRC_PATH/config.d/enable_zero_copy_replication.xml $DEST_SERVER_PATH/config.d/ ln -sf $SRC_PATH/config.d/nlp.xml $DEST_SERVER_PATH/config.d/ +ln -sf $SRC_PATH/config.d/enable_keeper_map.xml $DEST_SERVER_PATH/config.d/ ln -sf $SRC_PATH/users.d/log_queries.xml $DEST_SERVER_PATH/users.d/ ln -sf $SRC_PATH/users.d/readonly.xml $DEST_SERVER_PATH/users.d/ diff --git a/tests/fuzz/all.dict b/tests/fuzz/all.dict index dff62cd68a7..a147878da9b 100644 --- a/tests/fuzz/all.dict +++ b/tests/fuzz/all.dict @@ -763,7 +763,6 @@ "MINUTE" "MM" "mod" -"modelEvaluate" "MODIFY" "MODIFY COLUMN" "MODIFY ORDER BY" diff --git a/tests/fuzz/dictionaries/functions.dict b/tests/fuzz/dictionaries/functions.dict index cbcad3c05da..b90697f0c3d 100644 --- a/tests/fuzz/dictionaries/functions.dict +++ b/tests/fuzz/dictionaries/functions.dict @@ -469,7 +469,6 @@ "subtractSeconds" "alphaTokens" "negate" -"modelEvaluate" "file" "roundAge" "MACStringToOUI" diff --git a/tests/integration/ci-runner.py b/tests/integration/ci-runner.py index 562497fe8b0..4eab305358b 100755 --- a/tests/integration/ci-runner.py +++ b/tests/integration/ci-runner.py @@ -168,7 +168,8 @@ def clear_ip_tables_and_restart_daemons(): try: logging.info("Killing all alive docker containers") subprocess.check_output( - "timeout -s 9 10m docker kill $(docker ps -q)", shell=True + "timeout -s 9 10m docker ps --quiet | xargs --no-run-if-empty docker kill", + shell=True, ) except subprocess.CalledProcessError as err: logging.info("docker kill excepted: " + str(err)) @@ -176,7 +177,8 @@ def clear_ip_tables_and_restart_daemons(): try: logging.info("Removing all docker containers") subprocess.check_output( - "timeout -s 9 10m docker rm $(docker ps -a -q) --force", shell=True + "timeout -s 9 10m docker ps --all --quiet | xargs --no-run-if-empty docker rm --force", + shell=True, ) except subprocess.CalledProcessError as err: logging.info("docker rm excepted: " + str(err)) diff --git a/tests/integration/runner b/tests/integration/runner index e1b9a55b43e..c1b50ed4240 100755 --- a/tests/integration/runner +++ b/tests/integration/runner @@ -135,7 +135,7 @@ def check_args_and_update_paths(args): def docker_kill_handler_handler(signum, frame): subprocess.check_call( - 'docker kill $(docker ps -a -q --filter name={name} --format="{{{{.ID}}}}")'.format( + 'docker ps --all --quiet --filter name={name} --format="{{{{.ID}}}}"'.format( name=CONTAINER_NAME ), shell=True, @@ -350,7 +350,8 @@ if __name__ == "__main__": # randomizer, we should remove it after Sep 2022 try: subprocess.check_call( - f"docker volume ls -q | grep '{VOLUME_NAME}_.*_volume' | xargs --no-run-if-empty docker volume rm", + "docker volume rm $(docker volume ls -q | " + f"grep '{VOLUME_NAME}_.*_volume')", shell=True, ) except Exception as ex: @@ -404,7 +405,7 @@ if __name__ == "__main__": ) containers = subprocess.check_output( - f"docker ps -a -q --filter name={CONTAINER_NAME} --format={{{{.ID}}}}", + f"docker ps --all --quiet --filter name={CONTAINER_NAME} --format={{{{.ID}}}}", shell=True, universal_newlines=True, ).splitlines() diff --git a/tests/integration/test_catboost_model_config_reload/__init__.py b/tests/integration/test_catboost_evaluate/__init__.py similarity index 100% rename from tests/integration/test_catboost_model_config_reload/__init__.py rename to tests/integration/test_catboost_evaluate/__init__.py diff --git a/tests/integration/test_catboost_evaluate/config/models_config.xml b/tests/integration/test_catboost_evaluate/config/models_config.xml new file mode 100644 index 00000000000..f63df06ee26 --- /dev/null +++ b/tests/integration/test_catboost_evaluate/config/models_config.xml @@ -0,0 +1,3 @@ + + /etc/clickhouse-server/model/libcatboostmodel.so + diff --git a/tests/integration/test_catboost_evaluate/model/amazon_model.bin b/tests/integration/test_catboost_evaluate/model/amazon_model.bin new file mode 100644 index 00000000000..4a37fbec310 Binary files /dev/null and b/tests/integration/test_catboost_evaluate/model/amazon_model.bin differ diff --git a/tests/integration/test_catboost_model_config_reload/model/libcatboostmodel.so b/tests/integration/test_catboost_evaluate/model/libcatboostmodel.so similarity index 100% rename from tests/integration/test_catboost_model_config_reload/model/libcatboostmodel.so rename to tests/integration/test_catboost_evaluate/model/libcatboostmodel.so diff --git a/tests/integration/test_catboost_model_config_reload/model/model.bin b/tests/integration/test_catboost_evaluate/model/simple_model.bin similarity index 100% rename from tests/integration/test_catboost_model_config_reload/model/model.bin rename to tests/integration/test_catboost_evaluate/model/simple_model.bin diff --git a/tests/integration/test_catboost_evaluate/test.py b/tests/integration/test_catboost_evaluate/test.py new file mode 100644 index 00000000000..a0915977ab6 --- /dev/null +++ b/tests/integration/test_catboost_evaluate/test.py @@ -0,0 +1,402 @@ +import os +import sys +import time + +import pytest + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) + +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) + +instance = cluster.add_instance( + "instance", stay_alive=True, main_configs=["config/models_config.xml"] +) + + +@pytest.fixture(scope="module") +def ch_cluster(): + try: + cluster.start() + + os.system( + "docker cp {local} {cont_id}:{dist}".format( + local=os.path.join(SCRIPT_DIR, "model/."), + cont_id=instance.docker_id, + dist="/etc/clickhouse-server/model", + ) + ) + instance.restart_clickhouse() + + yield cluster + + finally: + cluster.shutdown() + + +# --------------------------------------------------------------------------- +# simple_model.bin has 2 float features and 9 categorical features + + +def testConstantFeatures(ch_cluster): + if instance.is_built_with_memory_sanitizer(): + pytest.skip("Memory Sanitizer cannot work with third-party shared libraries") + + result = instance.query("system reload models") + + result = instance.query( + "select catboostEvaluate('/etc/clickhouse-server/model/simple_model.bin', 1.0, 2.0, 3, 4, 5, 6, 7, 8, 9, 10, 11);" + ) + expected = "-1.930268705869267\n" + assert result == expected + + +def testNonConstantFeatures(ch_cluster): + if instance.is_built_with_memory_sanitizer(): + pytest.skip("Memory Sanitizer cannot work with third-party shared libraries") + + result = instance.query("system reload models") + + instance.query("DROP TABLE IF EXISTS T;") + instance.query( + "CREATE TABLE T(ID UInt32, F1 Float32, F2 Float32, F3 UInt32, F4 UInt32, F5 UInt32, F6 UInt32, F7 UInt32, F8 UInt32, F9 Float32, F10 Float32, F11 Float32) ENGINE MergeTree ORDER BY ID;" + ) + instance.query("INSERT INTO T VALUES(0, 1.0, 2.0, 3, 4, 5, 6, 7, 8, 9, 10, 11);") + + result = instance.query( + "select catboostEvaluate('/etc/clickhouse-server/model/simple_model.bin', F1, F2, F3, F4, F5, F6, F7, F8, F9, F10, F11) from T;" + ) + expected = "-1.930268705869267\n" + assert result == expected + + instance.query("DROP TABLE IF EXISTS T;") + + +def testModelPathIsNotAConstString(ch_cluster): + if instance.is_built_with_memory_sanitizer(): + pytest.skip("Memory Sanitizer cannot work with third-party shared libraries") + + result = instance.query("system reload models") + + err = instance.query_and_get_error( + "select catboostEvaluate(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11);" + ) + assert ( + "Illegal type UInt8 of first argument of function catboostEvaluate, expected a string" + in err + ) + + instance.query("DROP TABLE IF EXISTS T;") + instance.query("CREATE TABLE T(ID UInt32, A String) ENGINE MergeTree ORDER BY ID") + instance.query("INSERT INTO T VALUES(0, 'test');") + err = instance.query_and_get_error( + "select catboostEvaluate(A, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11) FROM T;" + ) + assert ( + "First argument of function catboostEvaluate must be a constant string" in err + ) + instance.query("DROP TABLE IF EXISTS T;") + + +def testWrongNumberOfFeatureArguments(ch_cluster): + if instance.is_built_with_memory_sanitizer(): + pytest.skip("Memory Sanitizer cannot work with third-party shared libraries") + + result = instance.query("system reload models") + + err = instance.query_and_get_error( + "select catboostEvaluate('/etc/clickhouse-server/model/simple_model.bin');" + ) + assert "Function catboostEvaluate expects at least 2 arguments" in err + + err = instance.query_and_get_error( + "select catboostEvaluate('/etc/clickhouse-server/model/simple_model.bin', 1, 2);" + ) + assert ( + "Number of columns is different with number of features: columns size 2 float features size 2 + cat features size 9" + in err + ) + + +def testFloatFeatureMustBeNumeric(ch_cluster): + if instance.is_built_with_memory_sanitizer(): + pytest.skip("Memory Sanitizer cannot work with third-party shared libraries") + + result = instance.query("system reload models") + + err = instance.query_and_get_error( + "select catboostEvaluate('/etc/clickhouse-server/model/simple_model.bin', 1.0, 'a', 3, 4, 5, 6, 7, 8, 9, 10, 11);" + ) + assert "Column 1 should be numeric to make float feature" in err + + +def testCategoricalFeatureMustBeNumericOrString(ch_cluster): + if instance.is_built_with_memory_sanitizer(): + pytest.skip("Memory Sanitizer cannot work with third-party shared libraries") + + result = instance.query("system reload models") + + err = instance.query_and_get_error( + "select catboostEvaluate('/etc/clickhouse-server/model/simple_model.bin', 1.0, 2.0, 3, 4, 5, 6, 7, tuple(8), 9, 10, 11);" + ) + assert "Column 7 should be numeric or string" in err + + +def testOnLowCardinalityFeatures(ch_cluster): + if instance.is_built_with_memory_sanitizer(): + pytest.skip("Memory Sanitizer cannot work with third-party shared libraries") + + result = instance.query("system reload models") + + # same but on domain-compressed data + result = instance.query( + "select catboostEvaluate('/etc/clickhouse-server/model/simple_model.bin', toLowCardinality(1.0), toLowCardinality(2.0), toLowCardinality(3), toLowCardinality(4), toLowCardinality(5), toLowCardinality(6), toLowCardinality(7), toLowCardinality(8), toLowCardinality(9), toLowCardinality(10), toLowCardinality(11));" + ) + expected = "-1.930268705869267\n" + assert result == expected + + +def testOnNullableFeatures(ch_cluster): + if instance.is_built_with_memory_sanitizer(): + pytest.skip("Memory Sanitizer cannot work with third-party shared libraries") + + result = instance.query("system reload models") + + result = instance.query( + "select catboostEvaluate('/etc/clickhouse-server/model/simple_model.bin', toNullable(1.0), toNullable(2.0), toNullable(3), toNullable(4), toNullable(5), toNullable(6), toNullable(7), toNullable(8), toNullable(9), toNullable(10), toNullable(11));" + ) + expected = "-1.930268705869267\n" + assert result == expected + + # Actual NULLs are disallowed + err = instance.query_and_get_error( + "select catboostEvaluate('/etc/clickhouse-server/model/simple_model.bin', toNullable(NULL), toNullable(NULL), toNullable(NULL), toNullable(NULL), toNullable(NULL), toNullable(NULL), toNullable(NULL), toNullable(NULL), toNullable(NULL), toNullable(NULL), toNullable(NULL));" + ) + assert "Column 0 should be numeric to make float feature" in err + + +def testInvalidLibraryPath(ch_cluster): + if instance.is_built_with_memory_sanitizer(): + pytest.skip("Memory Sanitizer cannot work with third-party shared libraries") + + result = instance.query("system reload models") + + # temporarily move library elsewhere + instance.exec_in_container( + [ + "bash", + "-c", + "mv /etc/clickhouse-server/model/libcatboostmodel.so /etc/clickhouse-server/model/nonexistant.so", + ] + ) + + err = instance.query_and_get_error( + "select catboostEvaluate('/etc/clickhouse-server/model/simple_model.bin', 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11);" + ) + assert ( + "Can't load library /etc/clickhouse-server/model/libcatboostmodel.so: file doesn't exist" + in err + ) + + # restore + instance.exec_in_container( + [ + "bash", + "-c", + "mv /etc/clickhouse-server/model/nonexistant.so /etc/clickhouse-server/model/libcatboostmodel.so", + ] + ) + + +def testInvalidModelPath(ch_cluster): + if instance.is_built_with_memory_sanitizer(): + pytest.skip("Memory Sanitizer cannot work with third-party shared libraries") + + result = instance.query("system reload models") + + err = instance.query_and_get_error( + "select catboostEvaluate('', 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11);" + ) + assert "Can't load model : file doesn't exist" in err + + err = instance.query_and_get_error( + "select catboostEvaluate('model_non_existant.bin', 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11);" + ) + assert "Can't load model model_non_existant.bin: file doesn't exist" in err + + +def testRecoveryAfterCrash(ch_cluster): + if instance.is_built_with_memory_sanitizer(): + pytest.skip("Memory Sanitizer cannot work with third-party shared libraries") + + result = instance.query("system reload models") + + result = instance.query( + "select catboostEvaluate('/etc/clickhouse-server/model/simple_model.bin', 1.0, 2.0, 3, 4, 5, 6, 7, 8, 9, 10, 11);" + ) + expected = "-1.930268705869267\n" + assert result == expected + + instance.exec_in_container( + ["bash", "-c", "kill -9 `pidof clickhouse-library-bridge`"], user="root" + ) + + result = instance.query( + "select catboostEvaluate('/etc/clickhouse-server/model/simple_model.bin', 1.0, 2.0, 3, 4, 5, 6, 7, 8, 9, 10, 11);" + ) + assert result == expected + + +# --------------------------------------------------------------------------- +# amazon_model.bin has 0 float features and 9 categorical features + + +def testAmazonModelSingleRow(ch_cluster): + if instance.is_built_with_memory_sanitizer(): + pytest.skip("Memory Sanitizer cannot work with third-party shared libraries") + + result = instance.query("system reload models") + + result = instance.query( + "select catboostEvaluate('/etc/clickhouse-server/model/amazon_model.bin', 1, 2, 3, 4, 5, 6, 7, 8, 9);" + ) + expected = "0.7774665009089274\n" + assert result == expected + + +def testAmazonModelManyRows(ch_cluster): + if instance.is_built_with_memory_sanitizer(): + pytest.skip("Memory Sanitizer cannot work with third-party shared libraries") + + result = instance.query("system reload models") + + result = instance.query("drop table if exists amazon") + + result = instance.query( + "create table amazon ( DATE Date materialized today(), ACTION UInt8, RESOURCE UInt32, MGR_ID UInt32, ROLE_ROLLUP_1 UInt32, ROLE_ROLLUP_2 UInt32, ROLE_DEPTNAME UInt32, ROLE_TITLE UInt32, ROLE_FAMILY_DESC UInt32, ROLE_FAMILY UInt32, ROLE_CODE UInt32) engine = MergeTree order by DATE" + ) + + result = instance.query( + "insert into amazon select number % 256, number, number, number, number, number, number, number, number, number from numbers(7500)" + ) + + # First compute prediction, then as a very crude way to fingerprint and compare the result: sum and floor + # (the focus is to test that the exchange of large result sets between the server and the bridge works) + result = instance.query( + "SELECT floor(sum(catboostEvaluate('/etc/clickhouse-server/model/amazon_model.bin', RESOURCE, MGR_ID, ROLE_ROLLUP_1, ROLE_ROLLUP_2, ROLE_DEPTNAME, ROLE_TITLE, ROLE_FAMILY_DESC, ROLE_FAMILY, ROLE_CODE))) FROM amazon" + ) + + expected = "5834\n" + assert result == expected + + result = instance.query("drop table if exists amazon") + + +def testModelUpdate(ch_cluster): + if instance.is_built_with_memory_sanitizer(): + pytest.skip("Memory Sanitizer cannot work with third-party shared libraries") + + result = instance.query("system reload models") + + query = "select catboostEvaluate('/etc/clickhouse-server/model/simple_model.bin', 1.0, 2.0, 3, 4, 5, 6, 7, 8, 9, 10, 11);" + + result = instance.query(query) + expected = "-1.930268705869267\n" + assert result == expected + + # simulate an update of the model: temporarily move the amazon model in place of the simple model + instance.exec_in_container( + [ + "bash", + "-c", + "mv /etc/clickhouse-server/model/simple_model.bin /etc/clickhouse-server/model/simple_model.bin.bak", + ] + ) + instance.exec_in_container( + [ + "bash", + "-c", + "mv /etc/clickhouse-server/model/amazon_model.bin /etc/clickhouse-server/model/simple_model.bin", + ] + ) + + # unload simple model + result = instance.query( + "system reload model '/etc/clickhouse-server/model/simple_model.bin'" + ) + + # load the simple-model-camouflaged amazon model + result = instance.query( + "select catboostEvaluate('/etc/clickhouse-server/model/simple_model.bin', 1, 2, 3, 4, 5, 6, 7, 8, 9);" + ) + expected = "0.7774665009089274\n" + assert result == expected + + # restore + instance.exec_in_container( + [ + "bash", + "-c", + "mv /etc/clickhouse-server/model/simple_model.bin /etc/clickhouse-server/model/amazon_model.bin", + ] + ) + instance.exec_in_container( + [ + "bash", + "-c", + "mv /etc/clickhouse-server/model/simple_model.bin.bak /etc/clickhouse-server/model/simple_model.bin", + ] + ) + + +def testSystemModelsAndModelRefresh(ch_cluster): + if instance.is_built_with_memory_sanitizer(): + pytest.skip("Memory Sanitizer cannot work with third-party shared libraries") + + result = instance.query("system reload models") + + # check model system view + result = instance.query("select * from system.models") + expected = "" + assert result == expected + + # load simple model + result = instance.query( + "select catboostEvaluate('/etc/clickhouse-server/model/simple_model.bin', 1.0, 2.0, 3, 4, 5, 6, 7, 8, 9, 10, 11);" + ) + expected = "-1.930268705869267\n" + assert result == expected + + # check model system view with one model loaded + result = instance.query("select * from system.models") + assert result.count("\n") == 1 + expected = "/etc/clickhouse-server/model/simple_model.bin" + assert expected in result + + # load amazon model + result = instance.query( + "select catboostEvaluate('/etc/clickhouse-server/model/amazon_model.bin', 1, 2, 3, 4, 5, 6, 7, 8, 9);" + ) + expected = "0.7774665009089274\n" + assert result == expected + + # check model system view with one model loaded + result = instance.query("select * from system.models") + assert result.count("\n") == 2 + expected = "/etc/clickhouse-server/model/simple_model.bin" + assert expected in result + expected = "/etc/clickhouse-server/model/amazon_model.bin" + assert expected in result + + # unload simple model + result = instance.query( + "system reload model '/etc/clickhouse-server/model/simple_model.bin'" + ) + + # check model system view, it should not display the removed model + result = instance.query("select * from system.models") + assert result.count("\n") == 1 + expected = "/etc/clickhouse-server/model/amazon_model.bin" + assert expected in result diff --git a/tests/integration/test_catboost_model_config_reload/config/catboost_lib.xml b/tests/integration/test_catboost_model_config_reload/config/catboost_lib.xml deleted file mode 100644 index 7aa06cc99ff..00000000000 --- a/tests/integration/test_catboost_model_config_reload/config/catboost_lib.xml +++ /dev/null @@ -1,3 +0,0 @@ - - /etc/clickhouse-server/model/libcatboostmodel.so - diff --git a/tests/integration/test_catboost_model_config_reload/config/models_config.xml b/tests/integration/test_catboost_model_config_reload/config/models_config.xml deleted file mode 100644 index 3cbf717bb67..00000000000 --- a/tests/integration/test_catboost_model_config_reload/config/models_config.xml +++ /dev/null @@ -1,2 +0,0 @@ - - diff --git a/tests/integration/test_catboost_model_config_reload/model/model_config.xml b/tests/integration/test_catboost_model_config_reload/model/model_config.xml deleted file mode 100644 index af9778097fa..00000000000 --- a/tests/integration/test_catboost_model_config_reload/model/model_config.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - catboost - model1 - /etc/clickhouse-server/model/model.bin - 0 - - diff --git a/tests/integration/test_catboost_model_config_reload/model/model_config2.xml b/tests/integration/test_catboost_model_config_reload/model/model_config2.xml deleted file mode 100644 index b81120ec900..00000000000 --- a/tests/integration/test_catboost_model_config_reload/model/model_config2.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - catboost - model2 - /etc/clickhouse-server/model/model.bin - 0 - - diff --git a/tests/integration/test_catboost_model_config_reload/test.py b/tests/integration/test_catboost_model_config_reload/test.py deleted file mode 100644 index c12c28e2338..00000000000 --- a/tests/integration/test_catboost_model_config_reload/test.py +++ /dev/null @@ -1,77 +0,0 @@ -import os -import sys -import time - -import pytest - -sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) - -from helpers.cluster import ClickHouseCluster - -cluster = ClickHouseCluster(__file__) -node = cluster.add_instance( - "node", - stay_alive=True, - main_configs=["config/models_config.xml", "config/catboost_lib.xml"], -) - - -def copy_file_to_container(local_path, dist_path, container_id): - os.system( - "docker cp {local} {cont_id}:{dist}".format( - local=local_path, cont_id=container_id, dist=dist_path - ) - ) - - -config = """ - /etc/clickhouse-server/model/{model_config} -""" - - -@pytest.fixture(scope="module") -def started_cluster(): - try: - cluster.start() - - copy_file_to_container( - os.path.join(SCRIPT_DIR, "model/."), - "/etc/clickhouse-server/model", - node.docker_id, - ) - node.restart_clickhouse() - - yield cluster - - finally: - cluster.shutdown() - - -def change_config(model_config): - node.replace_config( - "/etc/clickhouse-server/config.d/models_config.xml", - config.format(model_config=model_config), - ) - node.query("SYSTEM RELOAD CONFIG;") - - -def test(started_cluster): - if node.is_built_with_memory_sanitizer(): - pytest.skip("Memory Sanitizer cannot work with third-party shared libraries") - - # Set config with the path to the first model. - change_config("model_config.xml") - - node.query("SELECT modelEvaluate('model1', 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11);") - - # Change path to the second model in config. - change_config("model_config2.xml") - - # Check that the new model is loaded. - node.query("SELECT modelEvaluate('model2', 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11);") - - # Check that the old model was unloaded. - node.query_and_get_error( - "SELECT modelEvaluate('model1', 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11);" - ) diff --git a/tests/integration/test_catboost_model_first_evaluate/config/models_config.xml b/tests/integration/test_catboost_model_first_evaluate/config/models_config.xml deleted file mode 100644 index 26f5c4d57f6..00000000000 --- a/tests/integration/test_catboost_model_first_evaluate/config/models_config.xml +++ /dev/null @@ -1,4 +0,0 @@ - - /etc/clickhouse-server/model/libcatboostmodel.so - /etc/clickhouse-server/model/model_config.xml - diff --git a/tests/integration/test_catboost_model_first_evaluate/model/libcatboostmodel.so b/tests/integration/test_catboost_model_first_evaluate/model/libcatboostmodel.so deleted file mode 100755 index 388d9f887b4..00000000000 Binary files a/tests/integration/test_catboost_model_first_evaluate/model/libcatboostmodel.so and /dev/null differ diff --git a/tests/integration/test_catboost_model_first_evaluate/model/model.bin b/tests/integration/test_catboost_model_first_evaluate/model/model.bin deleted file mode 100644 index 118e099d176..00000000000 Binary files a/tests/integration/test_catboost_model_first_evaluate/model/model.bin and /dev/null differ diff --git a/tests/integration/test_catboost_model_first_evaluate/model/model_config.xml b/tests/integration/test_catboost_model_first_evaluate/model/model_config.xml deleted file mode 100644 index 2c328167a94..00000000000 --- a/tests/integration/test_catboost_model_first_evaluate/model/model_config.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - catboost - titanic - /etc/clickhouse-server/model/model.bin - 0 - - diff --git a/tests/integration/test_catboost_model_first_evaluate/test.py b/tests/integration/test_catboost_model_first_evaluate/test.py deleted file mode 100644 index b15f481c0e9..00000000000 --- a/tests/integration/test_catboost_model_first_evaluate/test.py +++ /dev/null @@ -1,48 +0,0 @@ -import os -import sys -import time - -import pytest - -sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) - -from helpers.cluster import ClickHouseCluster - -cluster = ClickHouseCluster(__file__) -node = cluster.add_instance( - "node", stay_alive=True, main_configs=["config/models_config.xml"] -) - - -def copy_file_to_container(local_path, dist_path, container_id): - os.system( - "docker cp {local} {cont_id}:{dist}".format( - local=local_path, cont_id=container_id, dist=dist_path - ) - ) - - -@pytest.fixture(scope="module") -def started_cluster(): - try: - cluster.start() - - copy_file_to_container( - os.path.join(SCRIPT_DIR, "model/."), - "/etc/clickhouse-server/model", - node.docker_id, - ) - node.restart_clickhouse() - - yield cluster - - finally: - cluster.shutdown() - - -def test(started_cluster): - if node.is_built_with_memory_sanitizer(): - pytest.skip("Memory Sanitizer cannot work with third-party shared libraries") - - node.query("select modelEvaluate('titanic', 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11);") diff --git a/tests/integration/test_catboost_model_reload/config/catboost_lib.xml b/tests/integration/test_catboost_model_reload/config/catboost_lib.xml deleted file mode 100644 index 7aa06cc99ff..00000000000 --- a/tests/integration/test_catboost_model_reload/config/catboost_lib.xml +++ /dev/null @@ -1,3 +0,0 @@ - - /etc/clickhouse-server/model/libcatboostmodel.so - diff --git a/tests/integration/test_catboost_model_reload/config/models_config.xml b/tests/integration/test_catboost_model_reload/config/models_config.xml deleted file mode 100644 index 84378df0e8f..00000000000 --- a/tests/integration/test_catboost_model_reload/config/models_config.xml +++ /dev/null @@ -1,3 +0,0 @@ - - /etc/clickhouse-server/model/model_config.xml - diff --git a/tests/integration/test_catboost_model_reload/model/conjunction.cbm b/tests/integration/test_catboost_model_reload/model/conjunction.cbm deleted file mode 100644 index 7b75fb5f886..00000000000 Binary files a/tests/integration/test_catboost_model_reload/model/conjunction.cbm and /dev/null differ diff --git a/tests/integration/test_catboost_model_reload/model/disjunction.cbm b/tests/integration/test_catboost_model_reload/model/disjunction.cbm deleted file mode 100644 index 8145c24637f..00000000000 Binary files a/tests/integration/test_catboost_model_reload/model/disjunction.cbm and /dev/null differ diff --git a/tests/integration/test_catboost_model_reload/model/libcatboostmodel.so b/tests/integration/test_catboost_model_reload/model/libcatboostmodel.so deleted file mode 100755 index 388d9f887b4..00000000000 Binary files a/tests/integration/test_catboost_model_reload/model/libcatboostmodel.so and /dev/null differ diff --git a/tests/integration/test_catboost_model_reload/model/model_config.xml b/tests/integration/test_catboost_model_reload/model/model_config.xml deleted file mode 100644 index 7cbda165ce9..00000000000 --- a/tests/integration/test_catboost_model_reload/model/model_config.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - catboost - model - /etc/clickhouse-server/model/model.cbm - 0 - - diff --git a/tests/integration/test_catboost_model_reload/test.py b/tests/integration/test_catboost_model_reload/test.py deleted file mode 100644 index 3bf7ca18cdd..00000000000 --- a/tests/integration/test_catboost_model_reload/test.py +++ /dev/null @@ -1,132 +0,0 @@ -import os -import sys -import time - -import pytest - -sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) - -from helpers.cluster import ClickHouseCluster - -cluster = ClickHouseCluster(__file__) -node = cluster.add_instance( - "node", - stay_alive=True, - main_configs=["config/models_config.xml", "config/catboost_lib.xml"], -) - - -def copy_file_to_container(local_path, dist_path, container_id): - os.system( - "docker cp {local} {cont_id}:{dist}".format( - local=local_path, cont_id=container_id, dist=dist_path - ) - ) - - -@pytest.fixture(scope="module") -def started_cluster(): - try: - cluster.start() - - copy_file_to_container( - os.path.join(SCRIPT_DIR, "model/."), - "/etc/clickhouse-server/model", - node.docker_id, - ) - node.query("CREATE TABLE binary (x UInt64, y UInt64) ENGINE = TinyLog()") - node.query("INSERT INTO binary VALUES (1, 1), (1, 0), (0, 1), (0, 0)") - - node.restart_clickhouse() - - yield cluster - - finally: - cluster.shutdown() - - -def test_model_reload(started_cluster): - if node.is_built_with_memory_sanitizer(): - pytest.skip("Memory Sanitizer cannot work with third-party shared libraries") - - node.exec_in_container( - ["bash", "-c", "rm -f /etc/clickhouse-server/model/model.cbm"] - ) - node.exec_in_container( - [ - "bash", - "-c", - "ln /etc/clickhouse-server/model/conjunction.cbm /etc/clickhouse-server/model/model.cbm", - ] - ) - node.query("SYSTEM RELOAD MODEL model") - - result = node.query( - """ - WITH modelEvaluate('model', toFloat64(x), toFloat64(y)) as prediction, exp(prediction) / (1 + exp(prediction)) as probability - SELECT if(probability > 0.5, 1, 0) FROM binary; - """ - ) - assert result == "1\n0\n0\n0\n" - - node.exec_in_container(["bash", "-c", "rm /etc/clickhouse-server/model/model.cbm"]) - node.exec_in_container( - [ - "bash", - "-c", - "ln /etc/clickhouse-server/model/disjunction.cbm /etc/clickhouse-server/model/model.cbm", - ] - ) - node.query("SYSTEM RELOAD MODEL model") - - result = node.query( - """ - WITH modelEvaluate('model', toFloat64(x), toFloat64(y)) as prediction, exp(prediction) / (1 + exp(prediction)) as probability - SELECT if(probability > 0.5, 1, 0) FROM binary; - """ - ) - assert result == "1\n1\n1\n0\n" - - -def test_models_reload(started_cluster): - if node.is_built_with_memory_sanitizer(): - pytest.skip("Memory Sanitizer cannot work with third-party shared libraries") - - node.exec_in_container( - ["bash", "-c", "rm -f /etc/clickhouse-server/model/model.cbm"] - ) - node.exec_in_container( - [ - "bash", - "-c", - "ln /etc/clickhouse-server/model/conjunction.cbm /etc/clickhouse-server/model/model.cbm", - ] - ) - node.query("SYSTEM RELOAD MODELS") - - result = node.query( - """ - WITH modelEvaluate('model', toFloat64(x), toFloat64(y)) as prediction, exp(prediction) / (1 + exp(prediction)) as probability - SELECT if(probability > 0.5, 1, 0) FROM binary; - """ - ) - assert result == "1\n0\n0\n0\n" - - node.exec_in_container(["bash", "-c", "rm /etc/clickhouse-server/model/model.cbm"]) - node.exec_in_container( - [ - "bash", - "-c", - "ln /etc/clickhouse-server/model/disjunction.cbm /etc/clickhouse-server/model/model.cbm", - ] - ) - node.query("SYSTEM RELOAD MODELS") - - result = node.query( - """ - WITH modelEvaluate('model', toFloat64(x), toFloat64(y)) as prediction, exp(prediction) / (1 + exp(prediction)) as probability - SELECT if(probability > 0.5, 1, 0) FROM binary; - """ - ) - assert result == "1\n1\n1\n0\n" diff --git a/tests/integration/test_join_set_family_s3/test.py b/tests/integration/test_join_set_family_s3/test.py index 38b56b7b15b..b09d5735628 100644 --- a/tests/integration/test_join_set_family_s3/test.py +++ b/tests/integration/test_join_set_family_s3/test.py @@ -27,7 +27,7 @@ def cluster(): def assert_objects_count(cluster, objects_count, path="data/"): minio = cluster.minio_client - s3_objects = list(minio.list_objects(cluster.minio_bucket, path, recursive=True)) + s3_objects = list(minio.list_objects(cluster.minio_bucket, path)) if objects_count != len(s3_objects): for s3_object in s3_objects: object_meta = minio.stat_object(cluster.minio_bucket, s3_object.object_name) diff --git a/tests/integration/test_keeper_incorrect_config/test.py b/tests/integration/test_keeper_incorrect_config/test.py index e0a28b00b4f..cedb195a6e0 100644 --- a/tests/integration/test_keeper_incorrect_config/test.py +++ b/tests/integration/test_keeper_incorrect_config/test.py @@ -172,6 +172,37 @@ NORMAL_CONFIG = """ """ +JUST_WRONG_CONFIG = """ + + + 9181 + 1 + /var/lib/clickhouse/coordination/log + /var/lib/clickhouse/coordination/snapshots + + + 5000 + 10000 + trace + + + + + 1 + node1 + 9234 + 2 + node2 + 9234 + 3 + node3 + 9234 + + + + +""" + def test_duplicate_endpoint(started_cluster): node1.stop_clickhouse() @@ -187,6 +218,7 @@ def test_duplicate_endpoint(started_cluster): assert_config_fails(DUPLICATE_ID_CONFIG) assert_config_fails(LOCALHOST_WITH_REMOTE) assert_config_fails(MULTIPLE_LOCAL_WITH_REMOTE) + assert_config_fails(JUST_WRONG_CONFIG) node1.replace_config( "/etc/clickhouse-server/config.d/enable_keeper1.xml", NORMAL_CONFIG diff --git a/tests/integration/test_s3_aws_sdk_is_total_garbage/__init__.py b/tests/integration/test_keeper_map/__init__.py similarity index 100% rename from tests/integration/test_s3_aws_sdk_is_total_garbage/__init__.py rename to tests/integration/test_keeper_map/__init__.py diff --git a/tests/integration/test_keeper_map/configs/enable_keeper_map.xml b/tests/integration/test_keeper_map/configs/enable_keeper_map.xml new file mode 100644 index 00000000000..b4cbb6a954b --- /dev/null +++ b/tests/integration/test_keeper_map/configs/enable_keeper_map.xml @@ -0,0 +1,3 @@ + + /test_keeper_map + diff --git a/tests/integration/test_keeper_map/test.py b/tests/integration/test_keeper_map/test.py new file mode 100644 index 00000000000..8f515077e8f --- /dev/null +++ b/tests/integration/test_keeper_map/test.py @@ -0,0 +1,179 @@ +import multiprocessing +import pytest +from time import sleep +import random +from itertools import count +from sys import stdout + +from multiprocessing import Pool + +from helpers.cluster import ClickHouseCluster +from helpers.test_tools import assert_eq_with_retry, assert_logs_contain +from helpers.network import PartitionManager + +test_recover_staled_replica_run = 1 + +cluster = ClickHouseCluster(__file__) + +node = cluster.add_instance( + "node", + main_configs=["configs/enable_keeper_map.xml"], + with_zookeeper=True, + stay_alive=True, +) + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + yield cluster + + finally: + cluster.shutdown() + + +def get_genuine_zk(): + return cluster.get_kazoo_client("zoo1") + + +def remove_children(client, path): + children = client.get_children(path) + + for child in children: + child_path = f"{path}/{child}" + remove_children(client, child_path) + client.delete(child_path) + + +def test_create_keeper_map(started_cluster): + node.query( + "CREATE TABLE test_keeper_map (key UInt64, value UInt64) ENGINE = KeeperMap('/test1') PRIMARY KEY(key);" + ) + zk_client = get_genuine_zk() + + def assert_children_size(path, expected_size): + assert len(zk_client.get_children(path)) == expected_size + + def assert_root_children_size(expected_size): + assert_children_size("/test_keeper_map/test1", expected_size) + + def assert_data_children_size(expected_size): + assert_children_size("/test_keeper_map/test1/data", expected_size) + + assert_root_children_size(2) + assert_data_children_size(0) + + node.query("INSERT INTO test_keeper_map VALUES (1, 11)") + assert_data_children_size(1) + + node.query( + "CREATE TABLE test_keeper_map_another (key UInt64, value UInt64) ENGINE = KeeperMap('/test1') PRIMARY KEY(key);" + ) + assert_root_children_size(2) + assert_data_children_size(1) + + node.query("INSERT INTO test_keeper_map_another VALUES (1, 11)") + assert_root_children_size(2) + assert_data_children_size(1) + + node.query("INSERT INTO test_keeper_map_another VALUES (2, 22)") + assert_root_children_size(2) + assert_data_children_size(2) + + node.query("DROP TABLE test_keeper_map SYNC") + assert_root_children_size(2) + assert_data_children_size(2) + + node.query("DROP TABLE test_keeper_map_another SYNC") + assert_root_children_size(0) + + zk_client.stop() + + +def create_drop_loop(index, stop_event): + table_name = f"test_keeper_map_{index}" + + for i in count(0, 1): + if stop_event.is_set(): + return + + node.query( + f"CREATE TABLE {table_name} (key UInt64, value UInt64) ENGINE = KeeperMap('/test') PRIMARY KEY(key);" + ) + node.query(f"INSERT INTO {table_name} VALUES ({index}, {i})") + result = node.query(f"SELECT value FROM {table_name} WHERE key = {index}") + assert result.strip() == str(i) + node.query(f"DROP TABLE {table_name} SYNC") + + +def test_create_drop_keeper_map_concurrent(started_cluster): + pool = Pool() + manager = multiprocessing.Manager() + stop_event = manager.Event() + results = [] + for i in range(multiprocessing.cpu_count()): + sleep(0.2) + results.append( + pool.apply_async( + create_drop_loop, + args=( + i, + stop_event, + ), + ) + ) + + sleep(60) + stop_event.set() + + for result in results: + result.get() + + pool.close() + + client = get_genuine_zk() + assert len(client.get_children("/test_keeper_map/test")) == 0 + client.stop() + + +def test_keeper_map_without_zk(started_cluster): + def assert_keeper_exception_after_partition(query): + with PartitionManager() as pm: + pm.drop_instance_zk_connections(node) + error = node.query_and_get_error(query) + assert "Coordination::Exception" in error + + assert_keeper_exception_after_partition( + "CREATE TABLE test_keeper_map (key UInt64, value UInt64) ENGINE = KeeperMap('/test1') PRIMARY KEY(key);" + ) + + node.query( + "CREATE TABLE test_keeper_map (key UInt64, value UInt64) ENGINE = KeeperMap('/test1') PRIMARY KEY(key);" + ) + + assert_keeper_exception_after_partition( + "INSERT INTO test_keeper_map VALUES (1, 11)" + ) + node.query("INSERT INTO test_keeper_map VALUES (1, 11)") + + assert_keeper_exception_after_partition("SELECT * FROM test_keeper_map") + node.query("SELECT * FROM test_keeper_map") + + with PartitionManager() as pm: + pm.drop_instance_zk_connections(node) + node.restart_clickhouse(60) + error = node.query_and_get_error("SELECT * FROM test_keeper_map") + assert "Failed to activate table because of connection issues" in error + + node.query("SELECT * FROM test_keeper_map") + + client = get_genuine_zk() + remove_children(client, "/test_keeper_map/test1") + node.restart_clickhouse(60) + error = node.query_and_get_error("SELECT * FROM test_keeper_map") + assert "Failed to activate table because of invalid metadata in ZooKeeper" in error + + node.query("DETACH TABLE test_keeper_map") + + client.stop() diff --git a/tests/integration/test_keeper_session/configs/keeper_config.xml b/tests/integration/test_keeper_session/configs/keeper_config1.xml similarity index 67% rename from tests/integration/test_keeper_session/configs/keeper_config.xml rename to tests/integration/test_keeper_session/configs/keeper_config1.xml index ed0bb52bd51..fd308fe8a2f 100644 --- a/tests/integration/test_keeper_session/configs/keeper_config.xml +++ b/tests/integration/test_keeper_session/configs/keeper_config1.xml @@ -1,4 +1,4 @@ - + 9181 1 @@ -19,9 +19,19 @@ 1 node1 9234 - true - 3 + + + 2 + node2 + 9234 + true + + + 3 + node3 + 9234 + true - + diff --git a/tests/integration/test_keeper_session/configs/keeper_config2.xml b/tests/integration/test_keeper_session/configs/keeper_config2.xml new file mode 100644 index 00000000000..ad558fbccad --- /dev/null +++ b/tests/integration/test_keeper_session/configs/keeper_config2.xml @@ -0,0 +1,37 @@ + + + 9181 + 2 + /var/lib/clickhouse/coordination/log + /var/lib/clickhouse/coordination/snapshots + * + + + 5000 + 10000 + 5000 + 75 + trace + + + + + 1 + node1 + 9234 + + + 2 + node2 + 9234 + true + + + 3 + node3 + 9234 + true + + + + diff --git a/tests/integration/test_keeper_session/configs/keeper_config3.xml b/tests/integration/test_keeper_session/configs/keeper_config3.xml new file mode 100644 index 00000000000..2a21f959816 --- /dev/null +++ b/tests/integration/test_keeper_session/configs/keeper_config3.xml @@ -0,0 +1,37 @@ + + + 9181 + 3 + /var/lib/clickhouse/coordination/log + /var/lib/clickhouse/coordination/snapshots + * + + + 5000 + 10000 + 5000 + 75 + trace + + + + + 1 + node1 + 9234 + + + 2 + node2 + 9234 + true + + + 3 + node3 + 9234 + true + + + + diff --git a/tests/integration/test_keeper_session/test.py b/tests/integration/test_keeper_session/test.py index 30db4d9548c..4b3aa7e3fdf 100644 --- a/tests/integration/test_keeper_session/test.py +++ b/tests/integration/test_keeper_session/test.py @@ -10,7 +10,15 @@ from kazoo.client import KazooClient cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance( - "node1", main_configs=["configs/keeper_config.xml"], stay_alive=True + "node1", main_configs=["configs/keeper_config1.xml"], stay_alive=True +) + +node2 = cluster.add_instance( + "node2", main_configs=["configs/keeper_config2.xml"], stay_alive=True +) + +node3 = cluster.add_instance( + "node3", main_configs=["configs/keeper_config3.xml"], stay_alive=True ) bool_struct = struct.Struct("B") @@ -61,7 +69,7 @@ def wait_node(node): def wait_nodes(): - for n in [node1]: + for n in [node1, node2, node3]: wait_node(n) @@ -165,3 +173,21 @@ def test_session_timeout(started_cluster): negotiated_timeout, _ = handshake(node1.name, session_timeout=20000, session_id=0) assert negotiated_timeout == 10000 + + +def test_session_close_shutdown(started_cluster): + wait_nodes() + + node1_zk = get_fake_zk(node1.name) + node2_zk = get_fake_zk(node2.name) + + eph_node = "/test_node" + node2_zk.create(eph_node, ephemeral=True) + assert node1_zk.exists(eph_node) != None + + # shutdown while session is active + node2.stop_clickhouse() + + assert node1_zk.exists(eph_node) == None + + node2.start_clickhouse() diff --git a/tests/integration/test_log_family_s3/test.py b/tests/integration/test_log_family_s3/test.py index bed379d098b..76ff0930db3 100644 --- a/tests/integration/test_log_family_s3/test.py +++ b/tests/integration/test_log_family_s3/test.py @@ -25,7 +25,7 @@ def cluster(): def assert_objects_count(cluster, objects_count, path="data/"): minio = cluster.minio_client - s3_objects = list(minio.list_objects(cluster.minio_bucket, path, recursive=True)) + s3_objects = list(minio.list_objects(cluster.minio_bucket, path)) if objects_count != len(s3_objects): for s3_object in s3_objects: object_meta = minio.stat_object(cluster.minio_bucket, s3_object.object_name) diff --git a/tests/integration/test_merge_tree_s3/configs/config.d/storage_conf.xml b/tests/integration/test_merge_tree_s3/configs/config.d/storage_conf.xml index 3ee49744a61..f3505f53339 100644 --- a/tests/integration/test_merge_tree_s3/configs/config.d/storage_conf.xml +++ b/tests/integration/test_merge_tree_s3/configs/config.d/storage_conf.xml @@ -38,6 +38,20 @@ /jbod1/ 1000000000 + + s3 + http://minio1:9001/root/data/ + minio + minio123 + 33554432 + + + cache + s3_r + /s3_cache_r/ + 1000000000 + 1 + @@ -78,6 +92,13 @@ + + +
+ s3_cache_r +
+ + diff --git a/tests/integration/test_merge_tree_s3/test.py b/tests/integration/test_merge_tree_s3/test.py index bee22c03689..4ce5fd5a069 100644 --- a/tests/integration/test_merge_tree_s3/test.py +++ b/tests/integration/test_merge_tree_s3/test.py @@ -6,7 +6,6 @@ import pytest from helpers.cluster import ClickHouseCluster from helpers.utility import generate_values, replace_config, SafeThread - SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) @@ -36,6 +35,7 @@ def cluster(): "/jbod1:size=2M", ], ) + logging.info("Starting cluster...") cluster.start() logging.info("Cluster started") @@ -121,17 +121,11 @@ def run_s3_mocks(cluster): def wait_for_delete_s3_objects(cluster, expected, timeout=30): minio = cluster.minio_client while timeout > 0: - if ( - len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) - == expected - ): + if len(list(minio.list_objects(cluster.minio_bucket, "data/"))) == expected: return timeout -= 1 time.sleep(1) - assert ( - len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) - == expected - ) + assert len(list(minio.list_objects(cluster.minio_bucket, "data/"))) == expected @pytest.fixture(autouse=True) @@ -147,9 +141,7 @@ def drop_table(cluster, node_name): wait_for_delete_s3_objects(cluster, 0) finally: # Remove extra objects to prevent tests cascade failing - for obj in list( - minio.list_objects(cluster.minio_bucket, "data/", recursive=True) - ): + for obj in list(minio.list_objects(cluster.minio_bucket, "data/")): minio.remove_object(cluster.minio_bucket, obj.object_name) @@ -171,7 +163,7 @@ def test_simple_insert_select( node.query("INSERT INTO s3_test VALUES {}".format(values1)) assert node.query("SELECT * FROM s3_test order by dt, id FORMAT Values") == values1 assert ( - len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) + len(list(minio.list_objects(cluster.minio_bucket, "data/"))) == FILES_OVERHEAD + files_per_part ) @@ -182,7 +174,7 @@ def test_simple_insert_select( == values1 + "," + values2 ) assert ( - len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) + len(list(minio.list_objects(cluster.minio_bucket, "data/"))) == FILES_OVERHEAD + files_per_part * 2 ) @@ -226,7 +218,7 @@ def test_insert_same_partition_and_merge(cluster, merge_vertical, node_name): node.query("SELECT count(distinct(id)) FROM s3_test FORMAT Values") == "(8192)" ) assert ( - len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) + len(list(minio.list_objects(cluster.minio_bucket, "data/"))) == FILES_OVERHEAD_PER_PART_WIDE * 6 + FILES_OVERHEAD ) @@ -315,28 +307,28 @@ def test_attach_detach_partition(cluster, node_name): ) assert node.query("SELECT count(*) FROM s3_test FORMAT Values") == "(8192)" assert ( - len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) + len(list(minio.list_objects(cluster.minio_bucket, "data/"))) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 2 ) node.query("ALTER TABLE s3_test DETACH PARTITION '2020-01-03'") assert node.query("SELECT count(*) FROM s3_test FORMAT Values") == "(4096)" assert ( - len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) + len(list(minio.list_objects(cluster.minio_bucket, "data/"))) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 2 ) node.query("ALTER TABLE s3_test ATTACH PARTITION '2020-01-03'") assert node.query("SELECT count(*) FROM s3_test FORMAT Values") == "(8192)" assert ( - len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) + len(list(minio.list_objects(cluster.minio_bucket, "data/"))) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 2 ) node.query("ALTER TABLE s3_test DROP PARTITION '2020-01-03'") assert node.query("SELECT count(*) FROM s3_test FORMAT Values") == "(4096)" assert ( - len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) + len(list(minio.list_objects(cluster.minio_bucket, "data/"))) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE ) @@ -347,8 +339,7 @@ def test_attach_detach_partition(cluster, node_name): ) assert node.query("SELECT count(*) FROM s3_test FORMAT Values") == "(0)" assert ( - len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) - == FILES_OVERHEAD + len(list(minio.list_objects(cluster.minio_bucket, "data/"))) == FILES_OVERHEAD ) @@ -366,21 +357,21 @@ def test_move_partition_to_another_disk(cluster, node_name): ) assert node.query("SELECT count(*) FROM s3_test FORMAT Values") == "(8192)" assert ( - len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) + len(list(minio.list_objects(cluster.minio_bucket, "data/"))) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 2 ) node.query("ALTER TABLE s3_test MOVE PARTITION '2020-01-04' TO DISK 'hdd'") assert node.query("SELECT count(*) FROM s3_test FORMAT Values") == "(8192)" assert ( - len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) + len(list(minio.list_objects(cluster.minio_bucket, "data/"))) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE ) node.query("ALTER TABLE s3_test MOVE PARTITION '2020-01-04' TO DISK 's3'") assert node.query("SELECT count(*) FROM s3_test FORMAT Values") == "(8192)" assert ( - len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) + len(list(minio.list_objects(cluster.minio_bucket, "data/"))) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 2 ) @@ -401,7 +392,7 @@ def test_table_manipulations(cluster, node_name): node.query("RENAME TABLE s3_test TO s3_renamed") assert node.query("SELECT count(*) FROM s3_renamed FORMAT Values") == "(8192)" assert ( - len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) + len(list(minio.list_objects(cluster.minio_bucket, "data/"))) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 2 ) node.query("RENAME TABLE s3_renamed TO s3_test") @@ -412,15 +403,14 @@ def test_table_manipulations(cluster, node_name): node.query("ATTACH TABLE s3_test") assert node.query("SELECT count(*) FROM s3_test FORMAT Values") == "(8192)" assert ( - len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) + len(list(minio.list_objects(cluster.minio_bucket, "data/"))) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 2 ) node.query("TRUNCATE TABLE s3_test") assert node.query("SELECT count(*) FROM s3_test FORMAT Values") == "(0)" assert ( - len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) - == FILES_OVERHEAD + len(list(minio.list_objects(cluster.minio_bucket, "data/"))) == FILES_OVERHEAD ) @@ -445,7 +435,7 @@ def test_move_replace_partition_to_another_table(cluster, node_name): assert node.query("SELECT sum(id) FROM s3_test FORMAT Values") == "(0)" assert node.query("SELECT count(*) FROM s3_test FORMAT Values") == "(16384)" assert ( - len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) + len(list(minio.list_objects(cluster.minio_bucket, "data/"))) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 4 ) @@ -459,7 +449,7 @@ def test_move_replace_partition_to_another_table(cluster, node_name): assert node.query("SELECT count(*) FROM s3_clone FORMAT Values") == "(8192)" # Number of objects in S3 should be unchanged. assert ( - len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) + len(list(minio.list_objects(cluster.minio_bucket, "data/"))) == FILES_OVERHEAD * 2 + FILES_OVERHEAD_PER_PART_WIDE * 4 ) @@ -473,7 +463,7 @@ def test_move_replace_partition_to_another_table(cluster, node_name): assert node.query("SELECT sum(id) FROM s3_test FORMAT Values") == "(0)" assert node.query("SELECT count(*) FROM s3_test FORMAT Values") == "(16384)" assert ( - len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) + len(list(minio.list_objects(cluster.minio_bucket, "data/"))) == FILES_OVERHEAD * 2 + FILES_OVERHEAD_PER_PART_WIDE * 6 ) @@ -494,14 +484,14 @@ def test_move_replace_partition_to_another_table(cluster, node_name): assert node.query("SELECT count(*) FROM s3_test FORMAT Values") == "(16384)" # Data should remain in S3 assert ( - len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) + len(list(minio.list_objects(cluster.minio_bucket, "data/"))) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 4 ) node.query("ALTER TABLE s3_test FREEZE") # Number S3 objects should be unchanged. assert ( - len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) + len(list(minio.list_objects(cluster.minio_bucket, "data/"))) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 4 ) @@ -510,7 +500,7 @@ def test_move_replace_partition_to_another_table(cluster, node_name): wait_for_delete_s3_objects(cluster, FILES_OVERHEAD_PER_PART_WIDE * 4) - for obj in list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True)): + for obj in list(minio.list_objects(cluster.minio_bucket, "data/")): minio.remove_object(cluster.minio_bucket, obj.object_name) @@ -531,7 +521,7 @@ def test_freeze_unfreeze(cluster, node_name): node.query("TRUNCATE TABLE s3_test") assert ( - len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) + len(list(minio.list_objects(cluster.minio_bucket, "data/"))) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 2 ) @@ -546,8 +536,7 @@ def test_freeze_unfreeze(cluster, node_name): # Data should be removed from S3. assert ( - len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) - == FILES_OVERHEAD + len(list(minio.list_objects(cluster.minio_bucket, "data/"))) == FILES_OVERHEAD ) @@ -570,7 +559,7 @@ def test_freeze_system_unfreeze(cluster, node_name): node.query("TRUNCATE TABLE s3_test") node.query("DROP TABLE s3_test_removed NO DELAY") assert ( - len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) + len(list(minio.list_objects(cluster.minio_bucket, "data/"))) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 2 ) @@ -581,8 +570,7 @@ def test_freeze_system_unfreeze(cluster, node_name): # Data should be removed from S3. assert ( - len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) - == FILES_OVERHEAD + len(list(minio.list_objects(cluster.minio_bucket, "data/"))) == FILES_OVERHEAD ) @@ -709,7 +697,7 @@ def test_lazy_seek_optimization_for_async_read(cluster, node_name): node.query("SELECT * FROM s3_test WHERE value LIKE '%abc%' ORDER BY value LIMIT 10") node.query("DROP TABLE IF EXISTS s3_test NO DELAY") minio = cluster.minio_client - for obj in list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True)): + for obj in list(minio.list_objects(cluster.minio_bucket, "data/")): minio.remove_object(cluster.minio_bucket, obj.object_name) @@ -754,3 +742,79 @@ def test_store_cleanup_disk_s3(cluster, node_name): "CREATE TABLE s3_test UUID '00000000-1000-4000-8000-000000000001' (n UInt64) Engine=MergeTree() ORDER BY n SETTINGS storage_policy='s3';" ) node.query("INSERT INTO s3_test SELECT 1") + + +@pytest.mark.parametrize("node_name", ["node"]) +def test_cache_setting_compatibility(cluster, node_name): + node = cluster.instances[node_name] + + node.query("DROP TABLE IF EXISTS s3_test NO DELAY") + + node.query( + "CREATE TABLE s3_test (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='s3_cache_r';" + ) + node.query( + "INSERT INTO s3_test SELECT * FROM generateRandom('key UInt32, value String') LIMIT 500" + ) + + result = node.query("SYSTEM DROP FILESYSTEM CACHE") + + result = node.query( + "SELECT count() FROM system.filesystem_cache WHERE cache_path LIKE '%persistent'" + ) + assert int(result) == 0 + + node.query("SELECT * FROM s3_test") + + result = node.query( + "SELECT count() FROM system.filesystem_cache WHERE cache_path LIKE '%persistent'" + ) + assert int(result) > 0 + + config_path = os.path.join( + SCRIPT_DIR, + f"./{cluster.instances_dir_name}/node/configs/config.d/storage_conf.xml", + ) + + replace_config( + config_path, + "1", + "0", + ) + + result = node.query("DESCRIBE CACHE 's3_cache_r'") + assert result.strip().endswith("1") + + node.restart_clickhouse() + + result = node.query("DESCRIBE CACHE 's3_cache_r'") + assert result.strip().endswith("0") + + result = node.query( + "SELECT count() FROM system.filesystem_cache WHERE cache_path LIKE '%persistent'" + ) + assert int(result) > 0 + + node.query("SELECT * FROM s3_test FORMAT Null") + + assert not node.contains_in_log("No such file or directory: Cache info:") + + replace_config( + config_path, + "0", + "1", + ) + + result = node.query( + "SELECT count() FROM system.filesystem_cache WHERE cache_path LIKE '%persistent'" + ) + assert int(result) > 0 + + node.restart_clickhouse() + + result = node.query("DESCRIBE CACHE 's3_cache_r'") + assert result.strip().endswith("1") + + node.query("SELECT * FROM s3_test FORMAT Null") + + assert not node.contains_in_log("No such file or directory: Cache info:") diff --git a/tests/integration/test_profile_events_s3/test.py b/tests/integration/test_profile_events_s3/test.py index 18f1c5ee9ad..a0f664df000 100644 --- a/tests/integration/test_profile_events_s3/test.py +++ b/tests/integration/test_profile_events_s3/test.py @@ -62,7 +62,7 @@ init_list = { def get_s3_events(instance): result = init_list.copy() events = instance.query( - "SELECT event, value FROM system.events WHERE event LIKE '%S3%'" + "SELECT event,value FROM system.events WHERE event LIKE '%S3%'" ).split("\n") for event in events: ev = event.split("\t") @@ -85,20 +85,20 @@ def get_minio_stat(cluster): ) ).text.split("\n") for line in stat: - x = re.search(r"s3_requests_total(\{.*\})?\s(\d+)(\s.*)?", line) + x = re.search("s3_requests_total(\{.*\})?\s(\d+)(\s.*)?", line) if x != None: y = re.search('.*api="(get|list|head|select).*', x.group(1)) if y != None: result["get_requests"] += int(x.group(2)) else: result["set_requests"] += int(x.group(2)) - x = re.search(r"s3_errors_total(\{.*\})?\s(\d+)(\s.*)?", line) + x = re.search("s3_errors_total(\{.*\})?\s(\d+)(\s.*)?", line) if x != None: result["errors"] += int(x.group(2)) - x = re.search(r"s3_rx_bytes_total(\{.*\})?\s([\d\.e\+\-]+)(\s.*)?", line) + x = re.search("s3_rx_bytes_total(\{.*\})?\s([\d\.e\+\-]+)(\s.*)?", line) if x != None: result["tx_bytes"] += float(x.group(2)) - x = re.search(r"s3_tx_bytes_total(\{.*\})?\s([\d\.e\+\-]+)(\s.*)?", line) + x = re.search("s3_tx_bytes_total(\{.*\})?\s([\d\.e\+\-]+)(\s.*)?", line) if x != None: result["rx_bytes"] += float(x.group(2)) return result @@ -128,10 +128,8 @@ def get_query_stat(instance, hint): def get_minio_size(cluster): minio = cluster.minio_client size = 0 - for obj_level1 in minio.list_objects( - cluster.minio_bucket, prefix="data/", recursive=True - ): - size += obj_level1.size + for obj in minio.list_objects(cluster.minio_bucket, "data/"): + size += obj.size return size @@ -147,7 +145,7 @@ def test_profile_events(cluster): metrics0 = get_s3_events(instance) minio0 = get_minio_stat(cluster) - query1 = "CREATE TABLE test_s3.test_s3 (key UInt32, value UInt32) ENGINE=MergeTree PRIMARY KEY key ORDER BY key SETTINGS storage_policy = 's3'" + query1 = "CREATE TABLE test_s3.test_s3 (key UInt32, value UInt32) ENGINE=MergeTree PRIMARY KEY key ORDER BY key SETTINGS storage_policy='s3'" instance.query(query1) size1 = get_minio_size(cluster) @@ -169,7 +167,7 @@ def test_profile_events(cluster): metrics1["WriteBufferFromS3Bytes"] - metrics0["WriteBufferFromS3Bytes"] == size1 ) - query2 = "INSERT INTO test_s3.test_s3 VALUES" + query2 = "INSERT INTO test_s3.test_s3 FORMAT Values" instance.query(query2 + " (1,1)") size2 = get_minio_size(cluster) @@ -184,12 +182,9 @@ def test_profile_events(cluster): metrics2["S3WriteRequestsCount"] - metrics1["S3WriteRequestsCount"] == minio2["set_requests"] - minio1["set_requests"] ) - stat2 = get_query_stat(instance, query2) - for metric in stat2: assert stat2[metric] == metrics2[metric] - metrics1[metric] - assert ( metrics2["WriteBufferFromS3Bytes"] - metrics1["WriteBufferFromS3Bytes"] == size2 - size1 @@ -210,7 +205,6 @@ def test_profile_events(cluster): == minio3["set_requests"] - minio2["set_requests"] ) stat3 = get_query_stat(instance, query3) - # With async reads profile events are not updated fully because reads are done in a separate thread. # for metric in stat3: # print(metric) diff --git a/tests/integration/test_replicated_merge_tree_s3/test.py b/tests/integration/test_replicated_merge_tree_s3/test.py index 0d978bb6967..37027d07969 100644 --- a/tests/integration/test_replicated_merge_tree_s3/test.py +++ b/tests/integration/test_replicated_merge_tree_s3/test.py @@ -113,7 +113,7 @@ def drop_table(cluster): minio = cluster.minio_client # Remove extra objects to prevent tests cascade failing - for obj in list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True)): + for obj in list(minio.list_objects(cluster.minio_bucket, "data/")): minio.remove_object(cluster.minio_bucket, obj.object_name) @@ -130,9 +130,9 @@ def test_insert_select_replicated(cluster, min_rows_for_wide_part, files_per_par insert(cluster, node_idxs=[1, 2, 3], verify=True) minio = cluster.minio_client - assert len( - list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True)) - ) == 3 * (FILES_OVERHEAD + files_per_part * 3) + assert len(list(minio.list_objects(cluster.minio_bucket, "data/"))) == 3 * ( + FILES_OVERHEAD + files_per_part * 3 + ) def test_drop_cache_on_cluster(cluster): diff --git a/tests/integration/test_replicated_merge_tree_s3_zero_copy/test.py b/tests/integration/test_replicated_merge_tree_s3_zero_copy/test.py index 60a1b9b9746..73b611ad169 100644 --- a/tests/integration/test_replicated_merge_tree_s3_zero_copy/test.py +++ b/tests/integration/test_replicated_merge_tree_s3_zero_copy/test.py @@ -87,7 +87,7 @@ def drop_table(cluster): minio = cluster.minio_client # Remove extra objects to prevent tests cascade failing - for obj in list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True)): + for obj in list(minio.list_objects(cluster.minio_bucket, "data/")): minio.remove_object(cluster.minio_bucket, obj.object_name) @@ -124,6 +124,6 @@ def test_insert_select_replicated(cluster, min_rows_for_wide_part, files_per_par ) minio = cluster.minio_client - assert len( - list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True)) - ) == (3 * FILES_OVERHEAD) + (files_per_part * 3) + assert len(list(minio.list_objects(cluster.minio_bucket, "data/"))) == ( + 3 * FILES_OVERHEAD + ) + (files_per_part * 3) diff --git a/tests/integration/test_s3_aws_sdk_has_slightly_unreliable_behaviour/__init__.py b/tests/integration/test_s3_aws_sdk_has_slightly_unreliable_behaviour/__init__.py new file mode 100644 index 00000000000..e5a0d9b4834 --- /dev/null +++ b/tests/integration/test_s3_aws_sdk_has_slightly_unreliable_behaviour/__init__.py @@ -0,0 +1 @@ +#!/usr/bin/env python3 diff --git a/tests/integration/test_s3_aws_sdk_is_total_garbage/configs/storage_conf.xml b/tests/integration/test_s3_aws_sdk_has_slightly_unreliable_behaviour/configs/storage_conf.xml similarity index 100% rename from tests/integration/test_s3_aws_sdk_is_total_garbage/configs/storage_conf.xml rename to tests/integration/test_s3_aws_sdk_has_slightly_unreliable_behaviour/configs/storage_conf.xml diff --git a/tests/integration/test_s3_aws_sdk_is_total_garbage/configs/upload_min_size.xml b/tests/integration/test_s3_aws_sdk_has_slightly_unreliable_behaviour/configs/upload_min_size.xml similarity index 100% rename from tests/integration/test_s3_aws_sdk_is_total_garbage/configs/upload_min_size.xml rename to tests/integration/test_s3_aws_sdk_has_slightly_unreliable_behaviour/configs/upload_min_size.xml diff --git a/tests/integration/test_s3_aws_sdk_is_total_garbage/s3_endpoint/endpoint.py b/tests/integration/test_s3_aws_sdk_has_slightly_unreliable_behaviour/s3_endpoint/endpoint.py similarity index 100% rename from tests/integration/test_s3_aws_sdk_is_total_garbage/s3_endpoint/endpoint.py rename to tests/integration/test_s3_aws_sdk_has_slightly_unreliable_behaviour/s3_endpoint/endpoint.py diff --git a/tests/integration/test_s3_aws_sdk_is_total_garbage/test.py b/tests/integration/test_s3_aws_sdk_has_slightly_unreliable_behaviour/test.py similarity index 100% rename from tests/integration/test_s3_aws_sdk_is_total_garbage/test.py rename to tests/integration/test_s3_aws_sdk_has_slightly_unreliable_behaviour/test.py diff --git a/tests/integration/test_s3_zero_copy_replication/test.py b/tests/integration/test_s3_zero_copy_replication/test.py index 860b83d4ed1..7b7fb9d21ad 100644 --- a/tests/integration/test_s3_zero_copy_replication/test.py +++ b/tests/integration/test_s3_zero_copy_replication/test.py @@ -39,9 +39,7 @@ def cluster(): def get_large_objects_count(cluster, size=100, folder="data"): minio = cluster.minio_client counter = 0 - for obj in minio.list_objects( - cluster.minio_bucket, "{}/".format(folder), recursive=True - ): + for obj in minio.list_objects(cluster.minio_bucket, "{}/".format(folder)): if obj.size is not None and obj.size >= size: counter = counter + 1 return counter diff --git a/tests/integration/test_send_crash_reports/test.py b/tests/integration/test_send_crash_reports/test.py index 90a6c684de7..83c0827f891 100644 --- a/tests/integration/test_send_crash_reports/test.py +++ b/tests/integration/test_send_crash_reports/test.py @@ -36,8 +36,10 @@ def started_node(): def test_send_segfault(started_node): + # NOTE: another option is to increase waiting time. if ( started_node.is_built_with_thread_sanitizer() + or started_node.is_built_with_address_sanitizer() or started_node.is_built_with_memory_sanitizer() ): pytest.skip("doesn't fit in timeouts for stacktrace generation") diff --git a/tests/performance/avg_weighted.xml b/tests/performance/avg_weighted.xml index df992ad682a..5aa89b08c35 100644 --- a/tests/performance/avg_weighted.xml +++ b/tests/performance/avg_weighted.xml @@ -32,5 +32,21 @@ SELECT avgWeighted(num_u, num) FROM perf_avg FORMAT Null SELECT avgWeighted(num_u, num_u) FROM perf_avg FORMAT Null + SELECT avgWeighted(num_f, num_f) FROM perf_avg FORMAT Null + SELECT avgWeighted(toNullable(num_f), num_f) FROM perf_avg FORMAT Null + SELECT avgWeighted(num_f, toNullable(num_f)) FROM perf_avg FORMAT Null + SELECT avgWeighted(toNullable(num_f), toNullable(num_f)) FROM perf_avg FORMAT Null + + SELECT avgWeightedIf(num_f, num_f, num % 10) FROM perf_avg FORMAT Null + SELECT avgWeightedIf(toNullable(num_f), num_f, num % 10) FROM perf_avg FORMAT Null + SELECT avgWeightedIf(num_f, toNullable(num_f), num % 10) FROM perf_avg FORMAT Null + SELECT avgWeightedIf(toNullable(num_f), toNullable(num_f), num % 10) FROM perf_avg FORMAT Null + + SELECT avgWeightedIf(num_f, num_f, toNullable(num) % 10) FROM perf_avg FORMAT Null + SELECT avgWeightedIf(toNullable(num_f), num_f, toNullable(num) % 10) FROM perf_avg FORMAT Null + SELECT avgWeightedIf(num_f, toNullable(num_f), toNullable(num) % 10) FROM perf_avg FORMAT Null + SELECT avgWeightedIf(toNullable(num_f), toNullable(num_f), toNullable(num) % 10) FROM perf_avg FORMAT Null + + DROP TABLE IF EXISTS perf_avg diff --git a/tests/performance/uniq_stored.xml b/tests/performance/uniq_stored.xml new file mode 100644 index 00000000000..75fb9847aab --- /dev/null +++ b/tests/performance/uniq_stored.xml @@ -0,0 +1,58 @@ + + + create table matview_1 + ( + a String, + b_count AggregateFunction(uniq, UInt64) + ) Engine=MergeTree partition by tuple() + ORDER by tuple() + SETTINGS index_granularity = 1024; + + + + create table matview_10000 + ( + a String, + b_count AggregateFunction(uniq, String) + ) Engine=MergeTree partition by tuple() + ORDER by tuple() + SETTINGS index_granularity = 1024; + + + + DROP TABLE IF EXISTS matview_1 + DROP TABLE IF EXISTS matview_10000 + + + INSERT INTO matview_10000 + SELECT a, uniqState(b) b_count + FROM + ( + SELECT toString(rand() % 1000) a, toString(number % 10000) b + FROM numbers_mt(20000000) + ) + GROUP BY a + SETTINGS max_insert_threads=8; + + OPTIMIZE TABLE matview_10000 FINAL + + + INSERT INTO matview_1 + SELECT '1', uniqState(number) b_count + FROM + ( + SELECT * + FROM numbers_mt(2000000) + ) + GROUP BY number + SETTINGS max_insert_threads=8; + + OPTIMIZE TABLE matview_1 FINAL + + + select a, uniqMerge(b_count) as b_count from matview_10000 prewhere a='55' group by a FORMAT Null SETTINGS max_threads=1; + select uniqMerge(b_count) as b_count from matview_10000 FORMAT Null SETTINGS max_threads=1; + + + select uniqMerge(b_count) as b_count FROM matview_1 FORMAT Null SETTINGS max_threads=1; + diff --git a/tests/queries/0_stateless/00632_get_sample_block_cache.sql b/tests/queries/0_stateless/00632_get_sample_block_cache.sql index cbb89387dd1..c54ca0b084e 100644 --- a/tests/queries/0_stateless/00632_get_sample_block_cache.sql +++ b/tests/queries/0_stateless/00632_get_sample_block_cache.sql @@ -1,4 +1,4 @@ --- Tags: long +-- Tags: long, no-s3-storage, no-asan SET joined_subquery_requires_alias = 0; diff --git a/tests/queries/0_stateless/01161_all_system_tables.sh b/tests/queries/0_stateless/01161_all_system_tables.sh index 9988c1f3625..1d886374c07 100755 --- a/tests/queries/0_stateless/01161_all_system_tables.sh +++ b/tests/queries/0_stateless/01161_all_system_tables.sh @@ -16,7 +16,7 @@ function run_selects() { thread_num=$1 readarray -t tables_arr < <(${CLICKHOUSE_CLIENT} -q "SELECT database || '.' || name FROM system.tables - WHERE database in ('system', 'information_schema', 'INFORMATION_SCHEMA') and name!='zookeeper' and name!='merge_tree_metadata_cache' + WHERE database in ('system', 'information_schema', 'INFORMATION_SCHEMA') and name!='zookeeper' and name!='merge_tree_metadata_cache' and name!='models' AND sipHash64(name || toString($RAND)) % $THREADS = $thread_num") for t in "${tables_arr[@]}" diff --git a/tests/queries/0_stateless/01321_monotonous_functions_in_order_by_bug.reference b/tests/queries/0_stateless/01321_monotonous_functions_in_order_by_bug.reference new file mode 100644 index 00000000000..0c720206065 --- /dev/null +++ b/tests/queries/0_stateless/01321_monotonous_functions_in_order_by_bug.reference @@ -0,0 +1,2 @@ +2020-01-01 01:00:00 1 +2020-01-01 01:00:00 999 diff --git a/tests/queries/0_stateless/01321_monotonous_functions_in_order_by_bug.sql b/tests/queries/0_stateless/01321_monotonous_functions_in_order_by_bug.sql new file mode 100644 index 00000000000..4aa52fe6ae8 --- /dev/null +++ b/tests/queries/0_stateless/01321_monotonous_functions_in_order_by_bug.sql @@ -0,0 +1,7 @@ +SELECT + toStartOfHour(c1) AS _c1, + c2 +FROM values((toDateTime('2020-01-01 01:01:01'), 999), (toDateTime('2020-01-01 01:01:59'), 1)) +ORDER BY + _c1 ASC, + c2 ASC diff --git a/tests/queries/0_stateless/01576_alias_column_rewrite.sql b/tests/queries/0_stateless/01576_alias_column_rewrite.sql index 8424eb11f9b..1f28225bef8 100644 --- a/tests/queries/0_stateless/01576_alias_column_rewrite.sql +++ b/tests/queries/0_stateless/01576_alias_column_rewrite.sql @@ -17,7 +17,7 @@ INSERT INTO test_table(timestamp, value) SELECT toDateTime('2020-01-01 12:00:00' INSERT INTO test_table(timestamp, value) SELECT toDateTime('2020-01-02 12:00:00'), 1 FROM numbers(10); INSERT INTO test_table(timestamp, value) SELECT toDateTime('2020-01-03 12:00:00'), 1 FROM numbers(10); -set optimize_respect_aliases = 1; +set optimize_respect_aliases = 1, optimize_monotonous_functions_in_order_by = 1; SELECT 'test-partition-prune'; SELECT COUNT() = 10 FROM test_table WHERE day = '2020-01-01' SETTINGS max_rows_to_read = 10; diff --git a/tests/queries/0_stateless/01646_rewrite_sum_if_bug.reference b/tests/queries/0_stateless/01646_rewrite_sum_if_bug.reference new file mode 100644 index 00000000000..dda2df4fd48 --- /dev/null +++ b/tests/queries/0_stateless/01646_rewrite_sum_if_bug.reference @@ -0,0 +1,2 @@ +67 +0 100 diff --git a/tests/queries/0_stateless/01646_rewrite_sum_if_bug.sql b/tests/queries/0_stateless/01646_rewrite_sum_if_bug.sql new file mode 100644 index 00000000000..3e6a7b92dbf --- /dev/null +++ b/tests/queries/0_stateless/01646_rewrite_sum_if_bug.sql @@ -0,0 +1,26 @@ +DROP TABLE IF EXISTS t; +create table t( s String ) Engine=Memory as select arrayJoin (['a','b','c']); + +SELECT round((sum(multiIf(s IN ('a', 'b'), 1, 0)) / count()) * 100) AS r +FROM cluster('test_cluster_two_shards', currentDatabase(), t); + +DROP TABLE t; + + +DROP TABLE IF EXISTS test_alias; + +CREATE TABLE test_alias(`a` Int64, `b` Int64, `c` Int64, `day` Date, `rtime` DateTime) ENGINE = Memory +as select 0, 0, 0, '2022-01-01', 0 from zeros(10); + +WITH + sum(if((a >= 0) AND (b != 100) AND (c = 0), 1, 0)) AS r1, + sum(if((a >= 0) AND (b != 100) AND (c > 220), 1, 0)) AS r2 +SELECT + (intDiv(toUInt32(rtime), 20) * 20) * 1000 AS t, + (r1 * 100) / (r1 + r2) AS m +FROM cluster('test_cluster_two_shards', currentDatabase(), test_alias) +WHERE day = '2022-01-01' +GROUP BY t +ORDER BY t ASC; + +DROP TABLE test_alias; diff --git a/tests/queries/0_stateless/01825_type_json_add_column.reference.j2 b/tests/queries/0_stateless/01825_type_json_add_column.reference.j2 new file mode 100644 index 00000000000..da724aef01a --- /dev/null +++ b/tests/queries/0_stateless/01825_type_json_add_column.reference.j2 @@ -0,0 +1,6 @@ +{% for storage in ["MergeTree", "ReplicatedMergeTree('/clickhouse/tables/{database}/test_01825_add_column/', 'r1')"] -%} +{"id":"1","s":{"k1":0}} +{"id":"2","s":{"k1":100}} +{"id":"1"} +{"id":"2"} +{% endfor -%} diff --git a/tests/queries/0_stateless/01825_type_json_add_column.sql.j2 b/tests/queries/0_stateless/01825_type_json_add_column.sql.j2 new file mode 100644 index 00000000000..87c76c042a6 --- /dev/null +++ b/tests/queries/0_stateless/01825_type_json_add_column.sql.j2 @@ -0,0 +1,23 @@ +-- Tags: no-fasttest + +{% for storage in ["MergeTree", "ReplicatedMergeTree('/clickhouse/tables/{database}/test_01825_add_column/', 'r1')"] -%} + +DROP TABLE IF EXISTS t_json_add_column; +SET allow_experimental_object_type = 1; + +CREATE TABLE t_json_add_column (id UInt64) ENGINE = {{ storage }} ORDER BY tuple(); + +INSERT INTO t_json_add_column VALUES (1); +ALTER TABLE t_json_add_column ADD COLUMN s JSON; + +INSERT INTO t_json_add_column VALUES(2, '{"k1": 100}'); + +SELECT * FROM t_json_add_column ORDER BY id FORMAT JSONEachRow; + +ALTER TABLE t_json_add_column DROP COLUMN s; + +SELECT * FROM t_json_add_column ORDER BY id FORMAT JSONEachRow; + +DROP TABLE t_json_add_column; + +{% endfor -%} diff --git a/tests/queries/0_stateless/01921_datatype_date32.reference b/tests/queries/0_stateless/01921_datatype_date32.reference index a33a96ffffb..b5bf4e06a4c 100644 --- a/tests/queries/0_stateless/01921_datatype_date32.reference +++ b/tests/queries/0_stateless/01921_datatype_date32.reference @@ -109,10 +109,10 @@ -------toStartOfFifteenMinutes--------- -------toStartOfHour--------- -------toStartOfISOYear--------- -2079-06-07 -2079-06-07 -2119-07-29 -2119-07-29 +1970-01-01 +1970-01-01 +2148-12-30 +2148-12-30 2021-01-04 -------toRelativeYearNum--------- 1900 diff --git a/tests/integration/test_catboost_model_first_evaluate/__init__.py b/tests/queries/0_stateless/02070_join_on_disk.reference similarity index 100% rename from tests/integration/test_catboost_model_first_evaluate/__init__.py rename to tests/queries/0_stateless/02070_join_on_disk.reference diff --git a/tests/queries/0_stateless/02070_join_on_disk.sql b/tests/queries/0_stateless/02070_join_on_disk.sql new file mode 100644 index 00000000000..eabf31df25f --- /dev/null +++ b/tests/queries/0_stateless/02070_join_on_disk.sql @@ -0,0 +1,21 @@ +-- Regression test when Join stores data on disk and receive empty block. +-- Because of this it does not create empty file, while expect it. + +SET max_threads = 1; +SET join_algorithm = 'auto'; +SET max_rows_in_join = 1000; +SET optimize_aggregation_in_order = 1; +SET max_block_size = 1000; + +DROP TABLE IF EXISTS join_on_disk; + +SYSTEM STOP MERGES join_on_disk; + +CREATE TABLE join_on_disk (id Int) Engine=MergeTree() ORDER BY id; + +INSERT INTO join_on_disk SELECT number as id FROM numbers_mt(50000); +INSERT INTO join_on_disk SELECT number as id FROM numbers_mt(1000); + +SELECT id FROM join_on_disk lhs LEFT JOIN (SELECT id FROM join_on_disk GROUP BY id) rhs USING (id) FORMAT Null; + +DROP TABLE join_on_disk; diff --git a/tests/queries/0_stateless/02117_show_create_table_system.reference b/tests/queries/0_stateless/02117_show_create_table_system.reference index 9e2f676bb55..d087bb55622 100644 --- a/tests/queries/0_stateless/02117_show_create_table_system.reference +++ b/tests/queries/0_stateless/02117_show_create_table_system.reference @@ -364,18 +364,6 @@ CREATE TABLE system.metrics ) ENGINE = SystemMetrics COMMENT 'SYSTEM TABLE is built on the fly.' -CREATE TABLE system.models -( - `name` String, - `status` Enum8('NOT_LOADED' = 0, 'LOADED' = 1, 'FAILED' = 2, 'LOADING' = 3, 'FAILED_AND_RELOADING' = 4, 'LOADED_AND_RELOADING' = 5, 'NOT_EXIST' = 6), - `origin` String, - `type` String, - `loading_start_time` DateTime, - `loading_duration` Float32, - `last_exception` String -) -ENGINE = SystemModels -COMMENT 'SYSTEM TABLE is built on the fly.' CREATE TABLE system.mutations ( `database` String, diff --git a/tests/queries/0_stateless/02117_show_create_table_system.sql b/tests/queries/0_stateless/02117_show_create_table_system.sql index 9a5726a0780..8b75ed60eec 100644 --- a/tests/queries/0_stateless/02117_show_create_table_system.sql +++ b/tests/queries/0_stateless/02117_show_create_table_system.sql @@ -45,7 +45,6 @@ show create table macros format TSVRaw; show create table merge_tree_settings format TSVRaw; show create table merges format TSVRaw; show create table metrics format TSVRaw; -show create table models format TSVRaw; show create table mutations format TSVRaw; show create table numbers format TSVRaw; show create table numbers_mt format TSVRaw; diff --git a/tests/queries/0_stateless/02149_read_in_order_fixed_prefix.sql b/tests/queries/0_stateless/02149_read_in_order_fixed_prefix.sql index 4dfcbb9bf80..44c1c12be35 100644 --- a/tests/queries/0_stateless/02149_read_in_order_fixed_prefix.sql +++ b/tests/queries/0_stateless/02149_read_in_order_fixed_prefix.sql @@ -56,7 +56,13 @@ ENGINE = MergeTree ORDER BY (toStartOfDay(dt), d); INSERT INTO t_read_in_order SELECT toDateTime('2020-10-10 00:00:00') + number, 1 / (number % 100 + 1), number FROM numbers(1000); EXPLAIN PIPELINE SELECT toStartOfDay(dt) as date, d FROM t_read_in_order ORDER BY date, round(d) LIMIT 5; -SELECT toStartOfDay(dt) as date, d FROM t_read_in_order ORDER BY date, round(d) LIMIT 5; +SELECT * from ( + SELECT toStartOfDay(dt) as date, d FROM t_read_in_order ORDER BY date, round(d) LIMIT 50000000000 + -- subquery with limit 50000000 to stabilize a test result and prevent order by d pushdown +) order by d limit 5; EXPLAIN PIPELINE SELECT toStartOfDay(dt) as date, d FROM t_read_in_order ORDER BY date, round(d) LIMIT 5; -SELECT toStartOfDay(dt) as date, d FROM t_read_in_order WHERE date = '2020-10-10' ORDER BY round(d) LIMIT 5; +SELECT * from ( + SELECT toStartOfDay(dt) as date, d FROM t_read_in_order WHERE date = '2020-10-10' ORDER BY round(d) LIMIT 50000000000 + -- subquery with limit 50000000 to stabilize a test result and prevent order by d pushdown +) order by d limit 5; diff --git a/tests/queries/0_stateless/02185_orc_corrupted_file.sh b/tests/queries/0_stateless/02185_orc_corrupted_file.sh index 7d7a714cccc..c5f5e8710ca 100755 --- a/tests/queries/0_stateless/02185_orc_corrupted_file.sh +++ b/tests/queries/0_stateless/02185_orc_corrupted_file.sh @@ -8,5 +8,4 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) USER_FILES_PATH=$(clickhouse-client --query "select _path,_file from file('nonexist.txt', 'CSV', 'val1 char')" 2>&1 | grep Exception | awk '{gsub("/nonexist.txt","",$9); print $9}') cp $CUR_DIR/data_orc/corrupted.orc $USER_FILES_PATH/ -${CLICKHOUSE_CLIENT} --query="select * from file('corrupted.orc')" 2>&1 | grep -F -q 'CANNOT_EXTRACT_TABLE_STRUCTURE' && echo 'OK' || echo 'FAIL' - +${CLICKHOUSE_CLIENT} --query="select * from file('corrupted.orc')" 2>&1 | grep -F -q 'Cannot extract table structure' && echo 'OK' || echo 'FAIL' diff --git a/tests/queries/0_stateless/02245_parquet_skip_unknown_type.sh b/tests/queries/0_stateless/02245_parquet_skip_unknown_type.sh index 005c089e434..1e416f23b69 100755 --- a/tests/queries/0_stateless/02245_parquet_skip_unknown_type.sh +++ b/tests/queries/0_stateless/02245_parquet_skip_unknown_type.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: no-fasttest +# Tags: no-fasttest, no-parallel CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh @@ -12,7 +12,6 @@ DATA_FILE=$USER_FILES_PATH/$FILE_NAME cp $CUR_DIR/data_parquet_bad_column/metadata_0.parquet $DATA_FILE -$CLICKHOUSE_CLIENT -q "desc file(test_02245.parquet)" 2>&1 | grep -qF "CANNOT_EXTRACT_TABLE_STRUCTURE" && echo "OK" || echo "FAIL" +$CLICKHOUSE_CLIENT -q "desc file(test_02245.parquet)" 2>&1 | grep -qF "Cannot extract table structure" && echo "OK" || echo "FAIL" $CLICKHOUSE_CLIENT -q "desc file(test_02245.parquet) settings input_format_parquet_skip_columns_with_unsupported_types_in_schema_inference=1" $CLICKHOUSE_CLIENT -q "select count(*) from file(test_02245.parquet) settings input_format_parquet_skip_columns_with_unsupported_types_in_schema_inference=1" - diff --git a/tests/queries/0_stateless/02267_file_globs_schema_inference.sql b/tests/queries/0_stateless/02267_file_globs_schema_inference.sql index 56b99d80286..6862d6f0602 100644 --- a/tests/queries/0_stateless/02267_file_globs_schema_inference.sql +++ b/tests/queries/0_stateless/02267_file_globs_schema_inference.sql @@ -1,4 +1,5 @@ -- Tags: no-fasttest, no-parallel + insert into function file('02267_data2.jsonl') select NULL as x; insert into function file('02267_data3.jsonl') select * from numbers(0); insert into function file('02267_data4.jsonl') select 1 as x; @@ -7,4 +8,4 @@ select * from file('02267_data*.jsonl') order by x; insert into function file('02267_data1.jsonl', 'TSV') select 1 as x; insert into function file('02267_data1.jsonl', 'TSV') select [1,2,3] as x; -select * from file('02267_data*.jsonl') settings schema_inference_use_cache_for_file=0; --{serverError CANNOT_EXTRACT_TABLE_STRUCTURE} +select * from file('02267_data*.jsonl') settings schema_inference_use_cache_for_file=0; --{serverError INCORRECT_DATA} diff --git a/tests/queries/0_stateless/02268_json_wrong_root_type_in_schema_inference.sql b/tests/queries/0_stateless/02268_json_wrong_root_type_in_schema_inference.sql index 9d435820ce2..7427426602a 100644 --- a/tests/queries/0_stateless/02268_json_wrong_root_type_in_schema_inference.sql +++ b/tests/queries/0_stateless/02268_json_wrong_root_type_in_schema_inference.sql @@ -1,6 +1,7 @@ +-- Tags: no-fasttest + insert into function file('02268_data.jsonl', 'TSV') select 1; -select * from file('02268_data.jsonl'); --{serverError CANNOT_EXTRACT_TABLE_STRUCTURE} +select * from file('02268_data.jsonl'); --{serverError 117} insert into function file('02268_data.jsonCompactEachRow', 'TSV') select 1; -select * from file('02268_data.jsonCompactEachRow'); --{serverError CANNOT_EXTRACT_TABLE_STRUCTURE} - +select * from file('02268_data.jsonCompactEachRow'); --{serverError 117} diff --git a/tests/queries/0_stateless/02286_mysql_dump_input_format.sh b/tests/queries/0_stateless/02286_mysql_dump_input_format.sh index f3253e0bf13..891734e9ad3 100755 --- a/tests/queries/0_stateless/02286_mysql_dump_input_format.sh +++ b/tests/queries/0_stateless/02286_mysql_dump_input_format.sh @@ -23,7 +23,7 @@ $CLICKHOUSE_CLIENT -q "desc file(dump1.sql, MySQLDump) settings input_format_mys $CLICKHOUSE_CLIENT -q "select * from file(dump1.sql, MySQLDump) settings input_format_mysql_dump_table_name='test'" $CLICKHOUSE_CLIENT -q "desc file(dump1.sql, MySQLDump) settings input_format_mysql_dump_table_name='test2'" $CLICKHOUSE_CLIENT -q "select * from file(dump1.sql, MySQLDump) settings input_format_mysql_dump_table_name='test2'" -$CLICKHOUSE_CLIENT -q "desc file(dump1.sql, MySQLDump) settings input_format_mysql_dump_table_name='test 3'" 2>&1 | grep -F -q 'CANNOT_EXTRACT_TABLE_STRUCTURE' && echo 'OK' || echo 'FAIL' +$CLICKHOUSE_CLIENT -q "desc file(dump1.sql, MySQLDump) settings input_format_mysql_dump_table_name='test 3'" 2>&1 | grep -F -q 'Cannot extract table structure' && echo 'OK' || echo 'FAIL' $CLICKHOUSE_CLIENT -q "select * from file(dump1.sql, MySQLDump, 'x Nullable(Int32)') settings input_format_mysql_dump_table_name='test 3'" 2>&1 | grep -F -q 'EMPTY_DATA_PASSED' && echo 'OK' || echo 'FAIL' echo "dump2" @@ -146,4 +146,3 @@ $CLICKHOUSE_CLIENT -q "desc file(dump15.sql, MySQLDump) settings input_format_my $CLICKHOUSE_CLIENT -q "select * from file(dump15.sql, MySQLDump) settings input_format_mysql_dump_table_name='test 3'" rm $USER_FILES_PATH/dump*.sql - diff --git a/tests/queries/0_stateless/02293_formats_json_columns.sh b/tests/queries/0_stateless/02293_formats_json_columns.sh index 74d9a4f5aab..7a21f8d9bab 100755 --- a/tests/queries/0_stateless/02293_formats_json_columns.sh +++ b/tests/queries/0_stateless/02293_formats_json_columns.sh @@ -88,6 +88,4 @@ echo ' } ' > $DATA_FILE -$CLICKHOUSE_CLIENT -q "desc file(data_02293, JSONColumns) settings input_format_max_rows_to_read_for_schema_inference=3" 2>&1 | grep -F -q 'CANNOT_EXTRACT_TABLE_STRUCTURE' && echo 'OK' || echo 'FAIL' - - +$CLICKHOUSE_CLIENT -q "desc file(data_02293, JSONColumns) settings input_format_max_rows_to_read_for_schema_inference=3" 2>&1 | grep -F -q 'Cannot extract table structure' && echo 'OK' || echo 'FAIL' diff --git a/tests/queries/0_stateless/02293_grouping_function.reference b/tests/queries/0_stateless/02293_grouping_function.reference index e71d6812ab5..7d745a0e0fa 100644 --- a/tests/queries/0_stateless/02293_grouping_function.reference +++ b/tests/queries/0_stateless/02293_grouping_function.reference @@ -8,7 +8,8 @@ GROUP BY (number), (number % 2) ) -ORDER BY number, gr; +ORDER BY number, gr +SETTINGS force_grouping_standard_compatibility=0; 0 1 0 1 0 2 @@ -30,7 +31,8 @@ GROUP BY (number), (number % 2) ) -ORDER BY number, gr; +ORDER BY number, gr +SETTINGS force_grouping_standard_compatibility=0; 0 1 0 2 0 2 @@ -52,7 +54,8 @@ GROUP BY (number), (number % 2) ) -ORDER BY number, gr; +ORDER BY number, gr +SETTINGS force_grouping_standard_compatibility=0; 0 0 0 1 0 1 @@ -73,7 +76,8 @@ GROUP BY (number), (number % 2) ) -ORDER BY number, grouping(number, number % 2) = 1; +ORDER BY number, grouping(number, number % 2) = 1 +SETTINGS force_grouping_standard_compatibility=0; 0 0 0 @@ -97,7 +101,8 @@ GROUP BY (number, number % 2), () ) -ORDER BY (gr, number); +ORDER BY (gr, number) +SETTINGS force_grouping_standard_compatibility=0; 0 10 0 0 1 2 1 1 2 @@ -129,7 +134,7 @@ GROUP BY ) HAVING grouping(number, number % 2) = 2 ORDER BY number -SETTINGS enable_optimize_predicate_expression = 0; +SETTINGS enable_optimize_predicate_expression = 0, force_grouping_standard_compatibility=0; 0 1 2 @@ -150,7 +155,7 @@ GROUP BY ) HAVING grouping(number, number % 2) = 1 ORDER BY number -SETTINGS enable_optimize_predicate_expression = 0; +SETTINGS enable_optimize_predicate_expression = 0, force_grouping_standard_compatibility=0; 0 0 SELECT @@ -161,7 +166,8 @@ GROUP BY GROUPING SETS ( (number), (number % 2)) -ORDER BY number, gr; +ORDER BY number, gr +SETTINGS force_grouping_standard_compatibility=0; 0 0 0 1 0 1 diff --git a/tests/queries/0_stateless/02293_grouping_function.sql b/tests/queries/0_stateless/02293_grouping_function.sql index 169fc09c324..cf076c8e51c 100644 --- a/tests/queries/0_stateless/02293_grouping_function.sql +++ b/tests/queries/0_stateless/02293_grouping_function.sql @@ -19,7 +19,8 @@ GROUP BY (number), (number % 2) ) -ORDER BY number, gr; +ORDER BY number, gr +SETTINGS force_grouping_standard_compatibility=0; SELECT number, @@ -30,7 +31,8 @@ GROUP BY (number), (number % 2) ) -ORDER BY number, gr; +ORDER BY number, gr +SETTINGS force_grouping_standard_compatibility=0; SELECT number, @@ -41,7 +43,8 @@ GROUP BY (number), (number % 2) ) -ORDER BY number, gr; +ORDER BY number, gr +SETTINGS force_grouping_standard_compatibility=0; SELECT number @@ -51,7 +54,8 @@ GROUP BY (number), (number % 2) ) -ORDER BY number, grouping(number, number % 2) = 1; +ORDER BY number, grouping(number, number % 2) = 1 +SETTINGS force_grouping_standard_compatibility=0; SELECT number, @@ -64,7 +68,8 @@ GROUP BY (number, number % 2), () ) -ORDER BY (gr, number); +ORDER BY (gr, number) +SETTINGS force_grouping_standard_compatibility=0; SELECT number @@ -76,7 +81,7 @@ GROUP BY ) HAVING grouping(number, number % 2) = 2 ORDER BY number -SETTINGS enable_optimize_predicate_expression = 0; +SETTINGS enable_optimize_predicate_expression = 0, force_grouping_standard_compatibility=0; SELECT number @@ -88,7 +93,7 @@ GROUP BY ) HAVING grouping(number, number % 2) = 1 ORDER BY number -SETTINGS enable_optimize_predicate_expression = 0; +SETTINGS enable_optimize_predicate_expression = 0, force_grouping_standard_compatibility=0; SELECT number, @@ -98,4 +103,5 @@ GROUP BY GROUPING SETS ( (number), (number % 2)) -ORDER BY number, gr; +ORDER BY number, gr +SETTINGS force_grouping_standard_compatibility=0; diff --git a/tests/queries/0_stateless/02293_grouping_function_group_by.reference b/tests/queries/0_stateless/02293_grouping_function_group_by.reference index 7f87aecd4bd..49cdca1411e 100644 --- a/tests/queries/0_stateless/02293_grouping_function_group_by.reference +++ b/tests/queries/0_stateless/02293_grouping_function_group_by.reference @@ -6,7 +6,8 @@ FROM remote('127.0.0.{2,3}', numbers(10)) GROUP BY number, number % 2 -ORDER BY number; +ORDER BY number +SETTINGS force_grouping_standard_compatibility=0; 0 1 1 1 2 1 @@ -25,7 +26,8 @@ FROM remote('127.0.0.{2,3}', numbers(10)) GROUP BY number, number % 2 -ORDER BY number; +ORDER BY number +SETTINGS force_grouping_standard_compatibility=0; 0 1 1 1 1 1 2 1 1 @@ -45,7 +47,8 @@ GROUP BY number % 2 WITH ROLLUP ORDER BY - number, gr; + number, gr +SETTINGS force_grouping_standard_compatibility=0; 0 0 0 2 0 3 @@ -74,7 +77,8 @@ FROM remote('127.0.0.{2,3}', numbers(10)) GROUP BY ROLLUP(number, number % 2) ORDER BY - number, gr; + number, gr +SETTINGS force_grouping_standard_compatibility=0; 0 0 0 2 0 3 @@ -105,7 +109,8 @@ GROUP BY number % 2 WITH CUBE ORDER BY - number, gr; + number, gr +SETTINGS force_grouping_standard_compatibility=0; 0 0 0 1 0 1 @@ -136,7 +141,8 @@ FROM remote('127.0.0.{2,3}', numbers(10)) GROUP BY CUBE(number, number % 2) ORDER BY - number, gr; + number, gr +SETTINGS force_grouping_standard_compatibility=0; 0 0 0 1 0 1 @@ -168,7 +174,8 @@ GROUP BY CUBE(number, number % 2) HAVING grouping(number) != 0 ORDER BY - number, gr; + number, gr +SETTINGS force_grouping_standard_compatibility=0; 0 5 0 6 1 5 @@ -205,7 +212,8 @@ FROM remote('127.0.0.{2,3}', numbers(10)) GROUP BY CUBE(number, number % 2) WITH TOTALS ORDER BY - number, gr; + number, gr +SETTINGS force_grouping_standard_compatibility=0; 0 0 0 1 0 1 @@ -247,7 +255,8 @@ FROM remote('127.0.0.{2,3}', numbers(10)) GROUP BY ROLLUP(number, number % 2) WITH TOTALS ORDER BY - number, gr; + number, gr +SETTINGS force_grouping_standard_compatibility=0; 0 0 0 2 0 3 diff --git a/tests/queries/0_stateless/02293_grouping_function_group_by.sql b/tests/queries/0_stateless/02293_grouping_function_group_by.sql index 9bf9d43478b..d438a8a5277 100644 --- a/tests/queries/0_stateless/02293_grouping_function_group_by.sql +++ b/tests/queries/0_stateless/02293_grouping_function_group_by.sql @@ -15,7 +15,8 @@ FROM remote('127.0.0.{2,3}', numbers(10)) GROUP BY number, number % 2 -ORDER BY number; +ORDER BY number +SETTINGS force_grouping_standard_compatibility=0; SELECT number, @@ -25,7 +26,8 @@ FROM remote('127.0.0.{2,3}', numbers(10)) GROUP BY number, number % 2 -ORDER BY number; +ORDER BY number +SETTINGS force_grouping_standard_compatibility=0; SELECT number, @@ -36,7 +38,8 @@ GROUP BY number % 2 WITH ROLLUP ORDER BY - number, gr; + number, gr +SETTINGS force_grouping_standard_compatibility=0; SELECT number, @@ -45,7 +48,8 @@ FROM remote('127.0.0.{2,3}', numbers(10)) GROUP BY ROLLUP(number, number % 2) ORDER BY - number, gr; + number, gr +SETTINGS force_grouping_standard_compatibility=0; SELECT number, @@ -56,7 +60,8 @@ GROUP BY number % 2 WITH CUBE ORDER BY - number, gr; + number, gr +SETTINGS force_grouping_standard_compatibility=0; SELECT number, @@ -65,7 +70,8 @@ FROM remote('127.0.0.{2,3}', numbers(10)) GROUP BY CUBE(number, number % 2) ORDER BY - number, gr; + number, gr +SETTINGS force_grouping_standard_compatibility=0; SELECT number, @@ -75,7 +81,8 @@ GROUP BY CUBE(number, number % 2) HAVING grouping(number) != 0 ORDER BY - number, gr; + number, gr +SETTINGS force_grouping_standard_compatibility=0; SELECT number, @@ -94,7 +101,8 @@ FROM remote('127.0.0.{2,3}', numbers(10)) GROUP BY CUBE(number, number % 2) WITH TOTALS ORDER BY - number, gr; + number, gr +SETTINGS force_grouping_standard_compatibility=0; SELECT number, @@ -113,4 +121,5 @@ FROM remote('127.0.0.{2,3}', numbers(10)) GROUP BY ROLLUP(number, number % 2) WITH TOTALS ORDER BY - number, gr; + number, gr +SETTINGS force_grouping_standard_compatibility=0; diff --git a/tests/queries/0_stateless/02315_grouping_constant_folding.reference b/tests/queries/0_stateless/02315_grouping_constant_folding.reference index 5aa979b1453..6e591de2661 100644 --- a/tests/queries/0_stateless/02315_grouping_constant_folding.reference +++ b/tests/queries/0_stateless/02315_grouping_constant_folding.reference @@ -1,5 +1,5 @@ -- { echoOn } -SELECT count() AS amount, a, b, GROUPING(a, b) FROM test02315 GROUP BY GROUPING SETS ((a, b), (a), ()) ORDER BY (amount, a, b); +SELECT count() AS amount, a, b, GROUPING(a, b) FROM test02315 GROUP BY GROUPING SETS ((a, b), (a), ()) ORDER BY (amount, a, b) SETTINGS force_grouping_standard_compatibility=0; 1 0 0 3 1 0 2 3 1 0 4 3 @@ -13,7 +13,7 @@ SELECT count() AS amount, a, b, GROUPING(a, b) FROM test02315 GROUP BY GROUPING 5 0 0 2 5 1 0 2 10 0 0 0 -SELECT count() AS amount, a, b, GROUPING(a, b) FROM test02315 GROUP BY ROLLUP(a, b) ORDER BY (amount, a, b); +SELECT count() AS amount, a, b, GROUPING(a, b) FROM test02315 GROUP BY ROLLUP(a, b) ORDER BY (amount, a, b) SETTINGS force_grouping_standard_compatibility=0; 1 0 0 3 1 0 2 3 1 0 4 3 diff --git a/tests/queries/0_stateless/02315_grouping_constant_folding.sql b/tests/queries/0_stateless/02315_grouping_constant_folding.sql index c4ef087a308..ff259b7be79 100644 --- a/tests/queries/0_stateless/02315_grouping_constant_folding.sql +++ b/tests/queries/0_stateless/02315_grouping_constant_folding.sql @@ -5,9 +5,9 @@ CREATE TABLE test02315(a UInt64, b UInt64) ENGINE=MergeTree() ORDER BY (a, b); INSERT INTO test02315 SELECT number % 2 as a, number as b FROM numbers(10); -- { echoOn } -SELECT count() AS amount, a, b, GROUPING(a, b) FROM test02315 GROUP BY GROUPING SETS ((a, b), (a), ()) ORDER BY (amount, a, b); +SELECT count() AS amount, a, b, GROUPING(a, b) FROM test02315 GROUP BY GROUPING SETS ((a, b), (a), ()) ORDER BY (amount, a, b) SETTINGS force_grouping_standard_compatibility=0; -SELECT count() AS amount, a, b, GROUPING(a, b) FROM test02315 GROUP BY ROLLUP(a, b) ORDER BY (amount, a, b); +SELECT count() AS amount, a, b, GROUPING(a, b) FROM test02315 GROUP BY ROLLUP(a, b) ORDER BY (amount, a, b) SETTINGS force_grouping_standard_compatibility=0; -- { echoOff } DROP TABLE test02315; diff --git a/tests/queries/0_stateless/02327_capnproto_protobuf_empty_messages.sh b/tests/queries/0_stateless/02327_capnproto_protobuf_empty_messages.sh index 9de01dbe294..69e65112305 100755 --- a/tests/queries/0_stateless/02327_capnproto_protobuf_empty_messages.sh +++ b/tests/queries/0_stateless/02327_capnproto_protobuf_empty_messages.sh @@ -15,11 +15,11 @@ mkdir -p $SCHEMADIR/$SERVER_SCHEMADIR cp -r $CLIENT_SCHEMADIR/02327_* $SCHEMADIR/$SERVER_SCHEMADIR/ -$CLICKHOUSE_CLIENT --query="desc file(data.pb) settings format_schema='$SERVER_SCHEMADIR/02327_schema:MessageWithEmpty'" 2>&1 | grep -F -q 'CANNOT_EXTRACT_TABLE_STRUCTURE' && echo 'OK' || echo 'FAIL'; -$CLICKHOUSE_CLIENT --query="desc file(data.capnp) settings format_schema='$SERVER_SCHEMADIR/02327_schema:MessageWithEmpty'" 2>&1 | grep -F -q 'CANNOT_EXTRACT_TABLE_STRUCTURE' && echo 'OK' || echo 'FAIL'; +$CLICKHOUSE_CLIENT --query="desc file(data.pb) settings format_schema='$SERVER_SCHEMADIR/02327_schema:MessageWithEmpty'" 2>&1 | grep -F -q 'Cannot extract table structure' && echo 'OK' || echo 'FAIL'; +$CLICKHOUSE_CLIENT --query="desc file(data.capnp) settings format_schema='$SERVER_SCHEMADIR/02327_schema:MessageWithEmpty'" 2>&1 | grep -F -q 'Cannot extract table structure' && echo 'OK' || echo 'FAIL'; -$CLICKHOUSE_CLIENT --query="create table test_protobuf engine=File(Protobuf) settings format_schema='$SERVER_SCHEMADIR/02327_schema:MessageWithEmpty'" 2>&1 | grep -F -q 'CANNOT_EXTRACT_TABLE_STRUCTURE' && echo 'OK' || echo 'FAIL'; -$CLICKHOUSE_CLIENT --query="create table test_capnp engine=File(CapnProto) settings format_schema='$SERVER_SCHEMADIR/02327_schema:MessageWithEmpty'" 2>&1 | grep -F -q 'CANNOT_EXTRACT_TABLE_STRUCTURE' && echo 'OK' || echo 'FAIL'; +$CLICKHOUSE_CLIENT --query="create table test_protobuf engine=File(Protobuf) settings format_schema='$SERVER_SCHEMADIR/02327_schema:MessageWithEmpty'" 2>&1 | grep -F -q 'Cannot extract table structure' && echo 'OK' || echo 'FAIL'; +$CLICKHOUSE_CLIENT --query="create table test_capnp engine=File(CapnProto) settings format_schema='$SERVER_SCHEMADIR/02327_schema:MessageWithEmpty'" 2>&1 | grep -F -q 'Cannot extract table structure' && echo 'OK' || echo 'FAIL'; $CLICKHOUSE_CLIENT --query="desc file(data.pb) settings format_schema='$SERVER_SCHEMADIR/02327_schema:MessageWithEmpty', input_format_protobuf_skip_fields_with_unsupported_types_in_schema_inference=1"; $CLICKHOUSE_CLIENT --query="desc file(data.capnp) settings format_schema='$SERVER_SCHEMADIR/02327_schema:MessageWithEmpty', input_format_capn_proto_skip_fields_with_unsupported_types_in_schema_inference=1"; diff --git a/tests/queries/0_stateless/02366_window_function_order_by.reference b/tests/queries/0_stateless/02366_window_function_order_by.reference new file mode 100644 index 00000000000..89f6c6c911a --- /dev/null +++ b/tests/queries/0_stateless/02366_window_function_order_by.reference @@ -0,0 +1,37 @@ +-- { echoOn } +SELECT groupArray(tuple(value)) OVER () +FROM (select number value from numbers(10)) +ORDER BY value ASC; +[(0),(1),(2),(3),(4),(5),(6),(7),(8),(9)] +[(0),(1),(2),(3),(4),(5),(6),(7),(8),(9)] +[(0),(1),(2),(3),(4),(5),(6),(7),(8),(9)] +[(0),(1),(2),(3),(4),(5),(6),(7),(8),(9)] +[(0),(1),(2),(3),(4),(5),(6),(7),(8),(9)] +[(0),(1),(2),(3),(4),(5),(6),(7),(8),(9)] +[(0),(1),(2),(3),(4),(5),(6),(7),(8),(9)] +[(0),(1),(2),(3),(4),(5),(6),(7),(8),(9)] +[(0),(1),(2),(3),(4),(5),(6),(7),(8),(9)] +[(0),(1),(2),(3),(4),(5),(6),(7),(8),(9)] +SELECT count() OVER (ORDER BY number + 1) FROM numbers(10) ORDER BY number; +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +SELECT count() OVER (ORDER BY number + 1) + 1 as foo FROM numbers(10) +ORDER BY foo; +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 diff --git a/tests/queries/0_stateless/02366_window_function_order_by.sql b/tests/queries/0_stateless/02366_window_function_order_by.sql new file mode 100644 index 00000000000..a3a02355cf7 --- /dev/null +++ b/tests/queries/0_stateless/02366_window_function_order_by.sql @@ -0,0 +1,9 @@ +-- { echoOn } +SELECT groupArray(tuple(value)) OVER () +FROM (select number value from numbers(10)) +ORDER BY value ASC; + +SELECT count() OVER (ORDER BY number + 1) FROM numbers(10) ORDER BY number; + +SELECT count() OVER (ORDER BY number + 1) + 1 as foo FROM numbers(10) +ORDER BY foo; diff --git a/tests/queries/0_stateless/02373_datetime64_monotonicity.reference b/tests/queries/0_stateless/02373_datetime64_monotonicity.reference new file mode 100644 index 00000000000..015f48a2b5e --- /dev/null +++ b/tests/queries/0_stateless/02373_datetime64_monotonicity.reference @@ -0,0 +1,3 @@ +10000000 +8358400 +8358400 diff --git a/tests/queries/0_stateless/02373_datetime64_monotonicity.sql b/tests/queries/0_stateless/02373_datetime64_monotonicity.sql new file mode 100644 index 00000000000..3b8982ab2a0 --- /dev/null +++ b/tests/queries/0_stateless/02373_datetime64_monotonicity.sql @@ -0,0 +1,14 @@ +drop table if exists dt64_monot_test sync; + +create table dt64_monot_test(date_time DateTime64(3), id String) + Engine=MergeTree + partition by toDate(date_time) + order by date_time; + +insert into dt64_monot_test select toDateTime64('2020-01-01 00:00:00.000',3)+number , '' from numbers(10000000); + +SELECT count() FROM dt64_monot_test; +SELECT count() FROM dt64_monot_test WHERE date_time >= toDateTime64('2020-01-20 00:00:00.000',3); +SELECT count() FROM dt64_monot_test WHERE toDateTime64(date_time,3) >= toDateTime64('2020-01-20 00:00:00.000',3); + +drop table if exists dt64_monot_test sync; \ No newline at end of file diff --git a/tests/queries/0_stateless/02373_datetime64_monotonicity1.queries b/tests/queries/0_stateless/02373_datetime64_monotonicity1.queries new file mode 100644 index 00000000000..212198c89de --- /dev/null +++ b/tests/queries/0_stateless/02373_datetime64_monotonicity1.queries @@ -0,0 +1,57 @@ +drop table if exists dt64_monot_test; +drop table if exists dt64_monot_test_string; +CREATE TABLE dt64_monot_test(`date_time` DateTime64(3, 'Europe/Berlin'), `id` String) ENGINE = MergeTree PARTITION BY toDate(date_time, 'Europe/Berlin') ORDER BY date_time; +insert into dt64_monot_test select toDateTime64('2020-01-01 00:00:00.000',3)+number , '' from numbers(10); + +SELECT count() FROM dt64_monot_test WHERE toDateTime(date_time) >= toDateTime('2020-01-01 00:00:00') SETTINGS force_index_by_date = 1, force_primary_key = 1; + +SELECT count()　FROM dt64_monot_test WHERE toDateTime64(date_time, 3) >= '2020-01-01 00:00:01.111' SETTINGS force_index_by_date = 1, force_primary_key = 1; + +SELECT count() FROM dt64_monot_test WHERE toDateTime64(date_time, 3) >= '2020-01-01 00:00:00.000' SETTINGS force_index_by_date = 1, force_primary_key = 1; + +SELECT count() FROM dt64_monot_test WHERE toDateTime64(date_time,3) >= toDateTime64('2020-01-01 00:00:00.000001', 3) SETTINGS force_index_by_date = 1, force_primary_key = 1; + +SELECT count() FROM dt64_monot_test WHERE toDateTime64(date_time,3) >= toDateTime64('2020-01-01 00:00:00.000001', 3, 'Europe/Berlin') SETTINGS force_index_by_date = 1, force_primary_key = 1; + +SELECT count() FROM dt64_monot_test WHERE toDateTime64(date_time,3) >= toDateTime64('2020-01-01 00:00:00.000001',6) SETTINGS force_index_by_date = 1; -- { serverError 277} + +SELECT count() FROM dt64_monot_test WHERE toDateTime64(date_time,3) >= toDateTime64('2020-01-01 00:00:00.000001',6, 'Europe/Berlin') SETTINGS force_primary_key = 1; -- { serverError 277} + +SELECT count() FROM dt64_monot_test WHERE toDateTime64(date_time,3) >= toDateTime64('2020-01-01 00:00:00.000001',6) SETTINGS force_primary_key = 1; -- { serverError 277} + +SELECT count() FROM dt64_monot_test WHERE toDateTime64(date_time,3) <= toDateTime64('2020-01-01 00:00:00.000001',3, 'Europe/Berlin') settings force_primary_key = 1; + +SELECT count() FROM dt64_monot_test WHERE toDateTime64(date_time,3) <= toDateTime64('2020-01-01 00:00:00.000001',3) settings force_primary_key = 1; + +SELECT count() FROM dt64_monot_test WHERE toDateTime64(date_time,3) = toDateTime64('2020-01-01 00:00:00.000000',6); + +SELECT count() FROM dt64_monot_test WHERE toDateTime64(date_time,3) = toDateTime64('2020-01-01 00:00:00.000000',6, 'Europe/Berlin'); + +SELECT count() FROM dt64_monot_test WHERE toDateTime64(date_time,6) = toDateTime64('2020-01-01 00:00:00.000000',6) settings force_index_by_date = 1, force_primary_key = 1; + +SELECT count() FROM dt64_monot_test WHERE toDateTime64(date_time,6) = toDateTime64('2020-01-01 00:00:00.000001',6, 'Europe/Berlin') settings force_index_by_date = 1, force_primary_key = 1; + +SELECT count() FROM dt64_monot_test WHERE toDateTime64(date_time,6) > toDateTime64('2020-01-01 00:00:00.000001',6, 'Europe/Berlin') settings force_index_by_date = 1, force_primary_key = 1; + +SELECT count() FROM dt64_monot_test WHERE toDateTime64(date_time,6) >= toDateTime64('2020-01-01 00:00:00.000001',6, 'Europe/Berlin') settings force_index_by_date = 1, force_primary_key = 1; + +SELECT count() FROM dt64_monot_test WHERE toDateTime64(date_time,6) >= toDateTime64('2020-01-01 00:00:00.000001',6) settings force_index_by_date = 1, force_primary_key = 1; + +SELECT count() FROM dt64_monot_test WHERE toDateTime64(date_time,0) >= toDateTime64('2020-01-01 00:00:00.000001',0, 'Europe/Berlin') settings force_index_by_date = 1, force_primary_key = 1; + +SELECT count() FROM dt64_monot_test WHERE toDateTime64(date_time,0) >= toDateTime64('2020-01-01 00:00:00.000001',0) settings force_index_by_date = 1, force_primary_key = 1; + +SELECT count() FROM dt64_monot_test WHERE toDateTime64(date_time,0) >= '2020-01-01 00:00:00' settings force_index_by_date = 1, force_primary_key = 1; + +SELECT count() FROM dt64_monot_test WHERE toDateTime64(date_time,0) >= '2020-01-01 00:00:01.1' settings force_index_by_date = 1, force_primary_key = 1; + +create table dt64_monot_test_string(date_time String, x String) Engine=MergeTree order by date_time; +insert into dt64_monot_test_string select '2020-01-01 00:00', '' from numbers(1); +insert into dt64_monot_test_string select '2020-01-01 00:00:00.000000' , '' from numbers(10); + +SELECT count() FROM dt64_monot_test_string WHERE toDateTime64(date_time,3) = '1970-01-01 00:00:00.000000000'; +SELECT count() FROM dt64_monot_test_string WHERE toDateTime64(date_time,3) = '1970-01-01 00:00:00.000000001'; +SELECT count() FROM dt64_monot_test_string WHERE toDateTime64(date_time,9) = '2020-01-01 00:00:00'; + +drop table dt64_monot_test; +drop table dt64_monot_test_string; diff --git a/tests/queries/0_stateless/02373_datetime64_monotonicity1.reference b/tests/queries/0_stateless/02373_datetime64_monotonicity1.reference new file mode 100644 index 00000000000..d9c310bdbc9 --- /dev/null +++ b/tests/queries/0_stateless/02373_datetime64_monotonicity1.reference @@ -0,0 +1,92 @@ +Asia/Tehran +10 +0 +0 +10 +0 +10 +1 +1 +0 +1 +0 +0 +0 +9 +0 +10 +0 +0 +0 +0 +10 + +UTC +10 +10 +10 +10 +10 +0 +1 +1 +0 +1 +0 +10 +10 +9 +10 +10 +10 +10 +1 +1 +10 + +Canada/Atlantic +10 +10 +10 +10 +10 +0 +1 +1 +0 +1 +0 +10 +10 +9 +10 +10 +10 +10 +0 +0 +10 + +Europe/Berlin +10 +8 +10 +10 +10 +1 +1 +1 +1 +1 +0 +9 +9 +9 +10 +10 +10 +9 +0 +0 +10 + diff --git a/tests/queries/0_stateless/02373_datetime64_monotonicity1.sh b/tests/queries/0_stateless/02373_datetime64_monotonicity1.sh new file mode 100755 index 00000000000..87c9ffcaaec --- /dev/null +++ b/tests/queries/0_stateless/02373_datetime64_monotonicity1.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +for tz in Asia/Tehran UTC Canada/Atlantic Europe/Berlin +do + echo "$tz" + TZ=$tz $CLICKHOUSE_LOCAL -mn < ${CUR_DIR}/02373_datetime64_monotonicity1.queries + echo "" +done diff --git a/tests/queries/0_stateless/02374_combine_multi_if_and_count_if_opt.sql b/tests/queries/0_stateless/02374_combine_multi_if_and_count_if_opt.sql index 06e1e4bfd55..c3367873042 100644 --- a/tests/queries/0_stateless/02374_combine_multi_if_and_count_if_opt.sql +++ b/tests/queries/0_stateless/02374_combine_multi_if_and_count_if_opt.sql @@ -4,6 +4,10 @@ create table m (a int) engine Log; insert into m values (1); +set optimize_rewrite_sum_if_to_count_if=1; + explain syntax select sum(multiIf(a = 1, 1, 0)) from m; +set optimize_rewrite_sum_if_to_count_if=0; + drop table m; diff --git a/tests/queries/0_stateless/02403_enable_extended_results_for_datetime_functions.reference b/tests/queries/0_stateless/02403_enable_extended_results_for_datetime_functions.reference new file mode 100644 index 00000000000..aa950215f59 --- /dev/null +++ b/tests/queries/0_stateless/02403_enable_extended_results_for_datetime_functions.reference @@ -0,0 +1,56 @@ +toStartOfYear;toDate32;true 1920-01-01 +type;toStartOfYear;toDate32;true Date32 +toStartOfYear;toDateTime64;true 1920-01-01 +type;toStartOfYear;toDateTime64;true Date32 +toStartOfISOYear;toDate32;true 1919-12-29 +type;toStartOfISOYear;toDate32;true Date32 +toStartOfISOYear;toDateTime64;true 1919-12-29 +type;toStartOfISOYear;toDateTime64;true Date32 +toStartOfQuarter;toDate32;true 1920-01-01 +type;toStartOfQuarter;toDate32;true Date32 +toStartOfQuarter;toDateTime64;true 1920-01-01 +type;toStartOfQuarter;toDateTime64;true Date32 +toStartOfMonth;toDate32;true 1920-02-01 +type;toStartOfMonth;toDate32;true Date32 +toStartOfMonth;toDateTime64;true 1920-02-01 +type;toStartOfMonth;toDateTime64;true Date32 +toStartOfWeek;toDate32;true 1920-02-01 +type;toStartOfWeek;toDate32;true Date32 +toStartOfWeek;toDateTime64;true 1920-02-01 +type;toStartOfWeek;toDateTime64;true Date32 +toMonday;toDate32;true 1920-02-02 +type;toMonday;toDate32;true Date32 +toMonday;toDateTime64;true 1920-02-02 +type;toMonday;toDateTime64;true Date32 +toLastDayOfMonth;toDate32;true 1920-02-29 +type;toLastDayOfMonth;toDate32;true Date32 +toLastDayOfMonth;toDateTime64;true 1920-02-29 +type;toLastDayOfMonth;toDateTime64;true Date32 +toStartOfYear;toDate32;false 1970-01-01 +type;toStartOfYear;toDate32;false Date +toStartOfYear;toDateTime64;false 1970-01-01 +type;toStartOfYear;toDateTime64;false Date +toStartOfISOYear;toDate32;false 1970-01-01 +type;toStartOfISOYear;toDate32;false Date +toStartOfISOYear;toDateTime64;false 1970-01-01 +type;toStartOfISOYear;toDateTime64;false Date +toStartOfQuarter;toDate32;false 1970-01-01 +type;toStartOfQuarter;toDate32;false Date +toStartOfQuarter;toDateTime64;false 1970-01-01 +type;toStartOfQuarter;toDateTime64;false Date +toStartOfMonth;toDate32;false 1970-01-01 +type;toStartOfMonth;toDate32;false Date +toStartOfMonth;toDateTime64;false 1970-01-01 +type;toStartOfMonth;toDateTime64;false Date +toStartOfWeek;toDate32;false 1970-01-01 +type;toStartOfWeek;toDate32;false Date +toStartOfWeek;toDateTime64;false 1970-01-01 +type;toStartOfWeek;toDateTime64;false Date +toMonday;toDate32;false 1970-01-01 +type;toMonday;toDate32;false Date +toMonday;toDateTime64;false 1970-01-01 +type;toMonday;toDateTime64;false Date +toLastDayOfMonth;toDate32;false 1970-01-01 +type;toLastDayOfMonth;toDate32;false Date +toLastDayOfMonth;toDateTime64;false 1970-01-01 +type;toLastDayOfMonth;toDateTime64;false Date diff --git a/tests/queries/0_stateless/02403_enable_extended_results_for_datetime_functions.sql.j2 b/tests/queries/0_stateless/02403_enable_extended_results_for_datetime_functions.sql.j2 new file mode 100644 index 00000000000..70c07c7792a --- /dev/null +++ b/tests/queries/0_stateless/02403_enable_extended_results_for_datetime_functions.sql.j2 @@ -0,0 +1,9 @@ +{% for option_value in ['true', 'false'] -%} +{% for date_fun in ['toStartOfYear', 'toStartOfISOYear', 'toStartOfQuarter', 'toStartOfMonth', 'toStartOfWeek', 'toMonday', 'toLastDayOfMonth'] -%} +SELECT '{{ date_fun }};toDate32;{{ option_value }}', {{ date_fun }}(toDate32('1920-02-02')) SETTINGS enable_extended_results_for_datetime_functions = {{ option_value }}; +SELECT 'type;{{ date_fun }};toDate32;{{ option_value }}', toTypeName({{ date_fun }}(toDate32('1920-02-02'))) SETTINGS enable_extended_results_for_datetime_functions = {{ option_value }}; +SELECT '{{ date_fun }};toDateTime64;{{ option_value }}', {{ date_fun }}(toDateTime64('1920-02-02 10:20:30', 3)) SETTINGS enable_extended_results_for_datetime_functions = {{ option_value }}; +SELECT 'type;{{ date_fun }};toDateTime64;{{ option_value }}', toTypeName({{ date_fun }}(toDateTime64('1920-02-02 10:20:30', 3))) SETTINGS enable_extended_results_for_datetime_functions = {{ option_value }}; +{% endfor -%} +{% endfor -%} + diff --git a/tests/queries/0_stateless/02413_model_evaluate_smoke.sql b/tests/queries/0_stateless/02413_model_evaluate_smoke.sql deleted file mode 100644 index 3b20067abfe..00000000000 --- a/tests/queries/0_stateless/02413_model_evaluate_smoke.sql +++ /dev/null @@ -1,2 +0,0 @@ --- This model does not exist: -SELECT modelEvaluate('hello', 1, 2, 3); -- { serverError 36 } diff --git a/tests/queries/0_stateless/02415_all_new_functions_must_be_documented.reference b/tests/queries/0_stateless/02415_all_new_functions_must_be_documented.reference index cbd92d0e8f4..6e0e41f11b8 100644 --- a/tests/queries/0_stateless/02415_all_new_functions_must_be_documented.reference +++ b/tests/queries/0_stateless/02415_all_new_functions_must_be_documented.reference @@ -192,6 +192,7 @@ caseWithExpr caseWithExpression caseWithoutExpr caseWithoutExpression +catboostEvaluate cbrt ceil char @@ -475,7 +476,6 @@ min2 minSampleSizeContinous minSampleSizeConversion minus -modelEvaluate modulo moduloLegacy moduloOrZero diff --git a/tests/queries/0_stateless/02416_grouping_function_compatibility.reference b/tests/queries/0_stateless/02416_grouping_function_compatibility.reference new file mode 100644 index 00000000000..c9a3ad2f593 --- /dev/null +++ b/tests/queries/0_stateless/02416_grouping_function_compatibility.reference @@ -0,0 +1,29 @@ +-- { echoOn } +SELECT count() AS amount, a, b, GROUPING(a, b) FROM test02416 GROUP BY GROUPING SETS ((a, b), (a), ()) ORDER BY (amount, a, b); +1 0 0 0 +1 0 2 0 +1 0 4 0 +1 0 6 0 +1 0 8 0 +1 1 1 0 +1 1 3 0 +1 1 5 0 +1 1 7 0 +1 1 9 0 +5 0 0 1 +5 1 0 1 +10 0 0 3 +SELECT count() AS amount, a, b, GROUPING(a, b) FROM test02416 GROUP BY ROLLUP(a, b) ORDER BY (amount, a, b); +1 0 0 0 +1 0 2 0 +1 0 4 0 +1 0 6 0 +1 0 8 0 +1 1 1 0 +1 1 3 0 +1 1 5 0 +1 1 7 0 +1 1 9 0 +5 0 0 1 +5 1 0 1 +10 0 0 3 diff --git a/tests/queries/0_stateless/02416_grouping_function_compatibility.sql b/tests/queries/0_stateless/02416_grouping_function_compatibility.sql new file mode 100644 index 00000000000..ed21055ade5 --- /dev/null +++ b/tests/queries/0_stateless/02416_grouping_function_compatibility.sql @@ -0,0 +1,14 @@ +DROP TABLE IF EXISTS test02416; + +CREATE TABLE test02416(a UInt64, b UInt64) ENGINE=MergeTree() ORDER BY (a, b); + +INSERT INTO test02416 SELECT number % 2 as a, number as b FROM numbers(10); + +-- { echoOn } +SELECT count() AS amount, a, b, GROUPING(a, b) FROM test02416 GROUP BY GROUPING SETS ((a, b), (a), ()) ORDER BY (amount, a, b); + +SELECT count() AS amount, a, b, GROUPING(a, b) FROM test02416 GROUP BY ROLLUP(a, b) ORDER BY (amount, a, b); + +-- { echoOff } +DROP TABLE test02416; + diff --git a/tests/queries/0_stateless/02416_keeper_map.reference b/tests/queries/0_stateless/02416_keeper_map.reference new file mode 100644 index 00000000000..eea8dd975e8 --- /dev/null +++ b/tests/queries/0_stateless/02416_keeper_map.reference @@ -0,0 +1,6 @@ +1 +1 +1 +1 +1 1 1 1 1 +1 diff --git a/tests/queries/0_stateless/02416_keeper_map.sql b/tests/queries/0_stateless/02416_keeper_map.sql new file mode 100644 index 00000000000..c191b539de6 --- /dev/null +++ b/tests/queries/0_stateless/02416_keeper_map.sql @@ -0,0 +1,38 @@ +-- Tags: no-ordinary-database, no-fasttest, long + +DROP TABLE IF EXISTS 02416_test SYNC; + +CREATE TABLE 02416_test (key String, value UInt32) Engine=KeeperMap('/' || currentDatabase() || '/test2416'); -- { serverError 36 } +CREATE TABLE 02416_test (key String, value UInt32) Engine=KeeperMap('/' || currentDatabase() || '/test2416') PRIMARY KEY(key2); -- { serverError 47 } +CREATE TABLE 02416_test (key String, value UInt32) Engine=KeeperMap('/' || currentDatabase() || '/test2416') PRIMARY KEY(key, value); -- { serverError 36 } +CREATE TABLE 02416_test (key String, value UInt32) Engine=KeeperMap('/' || currentDatabase() || '/test2416') PRIMARY KEY(concat(key, value)); -- { serverError 36 } +CREATE TABLE 02416_test (key Tuple(String, UInt32), value UInt64) Engine=KeeperMap('/' || currentDatabase() || '/test2416') PRIMARY KEY(key); + +DROP TABLE IF EXISTS 02416_test SYNC; +CREATE TABLE 02416_test (key String, value UInt32) Engine=KeeperMap('/' || currentDatabase() || '/test2416') PRIMARY KEY(key); + +INSERT INTO 02416_test SELECT '1_1', number FROM numbers(1000); +SELECT COUNT(1) == 1 FROM 02416_test; + +INSERT INTO 02416_test SELECT concat(toString(number), '_1'), number FROM numbers(1000); +SELECT COUNT(1) == 1000 FROM 02416_test; +SELECT uniqExact(key) == 32 FROM (SELECT * FROM 02416_test LIMIT 32 SETTINGS max_block_size = 1); +SELECT SUM(value) == 1 + 99 + 900 FROM 02416_test WHERE key IN ('1_1', '99_1', '900_1'); + +DROP TABLE IF EXISTS 02416_test SYNC; +DROP TABLE IF EXISTS 02416_test_memory; + +CREATE TABLE 02416_test (k UInt32, value UInt64, dummy Tuple(UInt32, Float64), bm AggregateFunction(groupBitmap, UInt64)) Engine=KeeperMap('/' || currentDatabase() || '/test2416') PRIMARY KEY(k); +CREATE TABLE 02416_test_memory AS 02416_test Engine = Memory; + +INSERT INTO 02416_test SELECT number % 77 AS k, SUM(number) AS value, (1, 1.2), bitmapBuild(groupArray(number)) FROM numbers(10000) group by k; + +INSERT INTO 02416_test_memory SELECT number % 77 AS k, SUM(number) AS value, (1, 1.2), bitmapBuild(groupArray(number)) FROM numbers(10000) group by k; + +SELECT A.a = B.a, A.b = B.b, A.c = B.c, A.d = B.d, A.e = B.e FROM ( SELECT 0 AS a, groupBitmapMerge(bm) AS b , SUM(k) AS c, SUM(value) AS d, SUM(dummy.1) AS e FROM 02416_test) A ANY LEFT JOIN (SELECT 0 AS a, groupBitmapMerge(bm) AS b , SUM(k) AS c, SUM(value) AS d, SUM(dummy.1) AS e FROM 02416_test_memory) B USING a ORDER BY a; + +TRUNCATE TABLE 02416_test; +SELECT 0 == COUNT(1) FROM 02416_test; + +DROP TABLE IF EXISTS 02416_test SYNC; +DROP TABLE IF EXISTS 02416_test_memory; diff --git a/tests/queries/0_stateless/02417_keeper_map_create_drop.reference b/tests/queries/0_stateless/02417_keeper_map_create_drop.reference new file mode 100644 index 00000000000..25bc8c288fb --- /dev/null +++ b/tests/queries/0_stateless/02417_keeper_map_create_drop.reference @@ -0,0 +1,10 @@ +1 11 +------ +1 11 +2 22 +------ +1 11 +2 22 +------ +1 11 +2 22 diff --git a/tests/queries/0_stateless/02417_keeper_map_create_drop.sql b/tests/queries/0_stateless/02417_keeper_map_create_drop.sql new file mode 100644 index 00000000000..49340167eaa --- /dev/null +++ b/tests/queries/0_stateless/02417_keeper_map_create_drop.sql @@ -0,0 +1,20 @@ +-- Tags: no-ordinary-database, no-fasttest + +DROP TABLE IF EXISTS 02417_test SYNC; + +CREATE TABLE 02417_test (key UInt64, value UInt64) Engine=KeeperMap('/' || currentDatabase() || '/test2417') PRIMARY KEY(key); +INSERT INTO 02417_test VALUES (1, 11); +SELECT * FROM 02417_test ORDER BY key; +SELECT '------'; + +CREATE TABLE 02417_test_another (key UInt64, value UInt64) Engine=KeeperMap('/' || currentDatabase() || '/test2417') PRIMARY KEY(key); +INSERT INTO 02417_test_another VALUES (2, 22); +SELECT * FROM 02417_test_another ORDER BY key; +SELECT '------'; +SELECT * FROM 02417_test ORDER BY key; +SELECT '------'; + +DROP TABLE 02417_test SYNC; +SELECT * FROM 02417_test_another ORDER BY key; + +DROP TABLE 02417_test_another SYNC; diff --git a/tests/queries/0_stateless/02417_null_variadic_behaviour.reference b/tests/queries/0_stateless/02417_null_variadic_behaviour.reference new file mode 100644 index 00000000000..bedb69f99b0 --- /dev/null +++ b/tests/queries/0_stateless/02417_null_variadic_behaviour.reference @@ -0,0 +1,65 @@ +-- { echo } +SELECT avgWeighted(number, number) t, toTypeName(t) FROM numbers(1); +nan Float64 +SELECT avgWeighted(number, number + 1) t, toTypeName(t) FROM numbers(0); +nan Float64 +SELECT avgWeighted(toNullable(number), number) t, toTypeName(t) FROM numbers(1); +nan Nullable(Float64) +SELECT avgWeighted(if(number < 10000, NULL, number), number) t, toTypeName(t) FROM numbers(100); +\N Nullable(Float64) +SELECT avgWeighted(if(number < 50, NULL, number), number) t, toTypeName(t) FROM numbers(100); +77.29530201342281 Nullable(Float64) +SELECT avgWeighted(number, if(number < 10000, NULL, number)) t, toTypeName(t) FROM numbers(100); +\N Nullable(Float64) +SELECT avgWeighted(number, if(number < 50, NULL, number)) t, toTypeName(t) FROM numbers(100); +77.29530201342281 Nullable(Float64) +SELECT avgWeighted(toNullable(number), if(number < 10000, NULL, number)) t, toTypeName(t) FROM numbers(100); +\N Nullable(Float64) +SELECT avgWeighted(toNullable(number), if(number < 50, NULL, number)) t, toTypeName(t) FROM numbers(100); +77.29530201342281 Nullable(Float64) +SELECT avgWeighted(if(number < 10000, NULL, number), toNullable(number)) t, toTypeName(t) FROM numbers(100); +\N Nullable(Float64) +SELECT avgWeighted(if(number < 50, NULL, number), toNullable(number)) t, toTypeName(t) FROM numbers(100); +77.29530201342281 Nullable(Float64) +SELECT avgWeighted(if(number < 10000, NULL, number), if(number < 10000, NULL, number)) t, toTypeName(t) FROM numbers(100); +\N Nullable(Float64) +SELECT avgWeighted(if(number < 50, NULL, number), if(number < 10000, NULL, number)) t, toTypeName(t) FROM numbers(100); +\N Nullable(Float64) +SELECT avgWeighted(if(number < 10000, NULL, number), if(number < 50, NULL, number)) t, toTypeName(t) FROM numbers(100); +\N Nullable(Float64) +SELECT avgWeighted(if(number < 50, NULL, number), if(number < 50, NULL, number)) t, toTypeName(t) FROM numbers(100); +77.29530201342281 Nullable(Float64) +SELECT avgWeightedIf(number, number, number % 10) t, toTypeName(t) FROM numbers(100); +66.63333333333334 Float64 +SELECT avgWeightedIf(number, number, toNullable(number % 10)) t, toTypeName(t) FROM numbers(100); +66.63333333333334 Float64 +SELECT avgWeightedIf(number, number, if(number < 10000, NULL, number % 10)) t, toTypeName(t) FROM numbers(100); +nan Float64 +SELECT avgWeightedIf(number, number, if(number < 50, NULL, number % 10)) t, toTypeName(t) FROM numbers(100); +77.75555555555556 Float64 +SELECT avgWeightedIf(number, number, if(number < 0, NULL, number % 10)) t, toTypeName(t) FROM numbers(100); +66.63333333333334 Float64 +SELECT avgWeightedIf(if(number < 10000, NULL, number), if(number < 10000, NULL, number), if(number < 10000, NULL, number % 10)) t, toTypeName(t) FROM numbers(100); +\N Nullable(Float64) +SELECT avgWeightedIf(if(number < 50, NULL, number), if(number < 10000, NULL, number), if(number < 10000, NULL, number % 10)) t, toTypeName(t) FROM numbers(100); +\N Nullable(Float64) +SELECT avgWeightedIf(if(number < 10000, NULL, number), if(number < 50, NULL, number), if(number < 10000, NULL, number % 10)) t, toTypeName(t) FROM numbers(100); +\N Nullable(Float64) +SELECT avgWeightedIf(if(number < 50, NULL, number), if(number < 50, NULL, number), if(number < 10000, NULL, number % 10)) t, toTypeName(t) FROM numbers(100); +\N Nullable(Float64) +SELECT avgWeightedIf(if(number < 10000, NULL, number), if(number < 10000, NULL, number), if(number < 50, NULL, number % 10)) t, toTypeName(t) FROM numbers(100); +\N Nullable(Float64) +SELECT avgWeightedIf(if(number < 50, NULL, number), if(number < 10000, NULL, number), if(number < 50, NULL, number % 10)) t, toTypeName(t) FROM numbers(100); +\N Nullable(Float64) +SELECT avgWeightedIf(if(number < 10000, NULL, number), if(number < 50, NULL, number), if(number < 50, NULL, number % 10)) t, toTypeName(t) FROM numbers(100); +\N Nullable(Float64) +SELECT avgWeightedIf(if(number < 50, NULL, number), if(number < 50, NULL, number), if(number < 50, NULL, number % 10)) t, toTypeName(t) FROM numbers(100); +77.75555555555556 Nullable(Float64) +SELECT avgWeightedIf(if(number < 10000, NULL, number), if(number < 10000, NULL, number), if(number < 0, NULL, number % 10)) t, toTypeName(t) FROM numbers(100); +\N Nullable(Float64) +SELECT avgWeightedIf(if(number < 50, NULL, number), if(number < 10000, NULL, number), if(number < 0, NULL, number % 10)) t, toTypeName(t) FROM numbers(100); +\N Nullable(Float64) +SELECT avgWeightedIf(if(number < 10000, NULL, number), if(number < 50, NULL, number), if(number < 0, NULL, number % 10)) t, toTypeName(t) FROM numbers(100); +\N Nullable(Float64) +SELECT avgWeightedIf(if(number < 50, NULL, number), if(number < 50, NULL, number), if(number < 0, NULL, number % 10)) t, toTypeName(t) FROM numbers(100); +77.75555555555556 Nullable(Float64) diff --git a/tests/queries/0_stateless/02417_null_variadic_behaviour.sql b/tests/queries/0_stateless/02417_null_variadic_behaviour.sql new file mode 100644 index 00000000000..566cf27bb90 --- /dev/null +++ b/tests/queries/0_stateless/02417_null_variadic_behaviour.sql @@ -0,0 +1,41 @@ +-- { echo } +SELECT avgWeighted(number, number) t, toTypeName(t) FROM numbers(1); +SELECT avgWeighted(number, number + 1) t, toTypeName(t) FROM numbers(0); + +SELECT avgWeighted(toNullable(number), number) t, toTypeName(t) FROM numbers(1); +SELECT avgWeighted(if(number < 10000, NULL, number), number) t, toTypeName(t) FROM numbers(100); +SELECT avgWeighted(if(number < 50, NULL, number), number) t, toTypeName(t) FROM numbers(100); + +SELECT avgWeighted(number, if(number < 10000, NULL, number)) t, toTypeName(t) FROM numbers(100); +SELECT avgWeighted(number, if(number < 50, NULL, number)) t, toTypeName(t) FROM numbers(100); + +SELECT avgWeighted(toNullable(number), if(number < 10000, NULL, number)) t, toTypeName(t) FROM numbers(100); +SELECT avgWeighted(toNullable(number), if(number < 50, NULL, number)) t, toTypeName(t) FROM numbers(100); +SELECT avgWeighted(if(number < 10000, NULL, number), toNullable(number)) t, toTypeName(t) FROM numbers(100); +SELECT avgWeighted(if(number < 50, NULL, number), toNullable(number)) t, toTypeName(t) FROM numbers(100); + +SELECT avgWeighted(if(number < 10000, NULL, number), if(number < 10000, NULL, number)) t, toTypeName(t) FROM numbers(100); +SELECT avgWeighted(if(number < 50, NULL, number), if(number < 10000, NULL, number)) t, toTypeName(t) FROM numbers(100); +SELECT avgWeighted(if(number < 10000, NULL, number), if(number < 50, NULL, number)) t, toTypeName(t) FROM numbers(100); +SELECT avgWeighted(if(number < 50, NULL, number), if(number < 50, NULL, number)) t, toTypeName(t) FROM numbers(100); + +SELECT avgWeightedIf(number, number, number % 10) t, toTypeName(t) FROM numbers(100); +SELECT avgWeightedIf(number, number, toNullable(number % 10)) t, toTypeName(t) FROM numbers(100); +SELECT avgWeightedIf(number, number, if(number < 10000, NULL, number % 10)) t, toTypeName(t) FROM numbers(100); +SELECT avgWeightedIf(number, number, if(number < 50, NULL, number % 10)) t, toTypeName(t) FROM numbers(100); +SELECT avgWeightedIf(number, number, if(number < 0, NULL, number % 10)) t, toTypeName(t) FROM numbers(100); + +SELECT avgWeightedIf(if(number < 10000, NULL, number), if(number < 10000, NULL, number), if(number < 10000, NULL, number % 10)) t, toTypeName(t) FROM numbers(100); +SELECT avgWeightedIf(if(number < 50, NULL, number), if(number < 10000, NULL, number), if(number < 10000, NULL, number % 10)) t, toTypeName(t) FROM numbers(100); +SELECT avgWeightedIf(if(number < 10000, NULL, number), if(number < 50, NULL, number), if(number < 10000, NULL, number % 10)) t, toTypeName(t) FROM numbers(100); +SELECT avgWeightedIf(if(number < 50, NULL, number), if(number < 50, NULL, number), if(number < 10000, NULL, number % 10)) t, toTypeName(t) FROM numbers(100); + +SELECT avgWeightedIf(if(number < 10000, NULL, number), if(number < 10000, NULL, number), if(number < 50, NULL, number % 10)) t, toTypeName(t) FROM numbers(100); +SELECT avgWeightedIf(if(number < 50, NULL, number), if(number < 10000, NULL, number), if(number < 50, NULL, number % 10)) t, toTypeName(t) FROM numbers(100); +SELECT avgWeightedIf(if(number < 10000, NULL, number), if(number < 50, NULL, number), if(number < 50, NULL, number % 10)) t, toTypeName(t) FROM numbers(100); +SELECT avgWeightedIf(if(number < 50, NULL, number), if(number < 50, NULL, number), if(number < 50, NULL, number % 10)) t, toTypeName(t) FROM numbers(100); + +SELECT avgWeightedIf(if(number < 10000, NULL, number), if(number < 10000, NULL, number), if(number < 0, NULL, number % 10)) t, toTypeName(t) FROM numbers(100); +SELECT avgWeightedIf(if(number < 50, NULL, number), if(number < 10000, NULL, number), if(number < 0, NULL, number % 10)) t, toTypeName(t) FROM numbers(100); +SELECT avgWeightedIf(if(number < 10000, NULL, number), if(number < 50, NULL, number), if(number < 0, NULL, number % 10)) t, toTypeName(t) FROM numbers(100); +SELECT avgWeightedIf(if(number < 50, NULL, number), if(number < 50, NULL, number), if(number < 0, NULL, number % 10)) t, toTypeName(t) FROM numbers(100); diff --git a/tests/queries/0_stateless/02417_opentelemetry_insert_on_distributed_table.reference b/tests/queries/0_stateless/02417_opentelemetry_insert_on_distributed_table.reference new file mode 100644 index 00000000000..dde07d4540d --- /dev/null +++ b/tests/queries/0_stateless/02417_opentelemetry_insert_on_distributed_table.reference @@ -0,0 +1,8 @@ +{"operation_name":"void DB::DistributedSink::writeToLocal(const Cluster::ShardInfo &, const DB::Block &, size_t)","cluster":"test_cluster_two_shards_localhost","shard":"1","rows":"1","bytes":"8"} +{"operation_name":"void DB::DistributedSink::writeToLocal(const Cluster::ShardInfo &, const DB::Block &, size_t)","cluster":"test_cluster_two_shards_localhost","shard":"2","rows":"1","bytes":"8"} +{"operation_name":"void DB::StorageDistributedDirectoryMonitor::processFile(const std::string &)","cluster":"test_cluster_two_shards_localhost","shard":"1","rows":"1","bytes":"8"} +{"operation_name":"void DB::StorageDistributedDirectoryMonitor::processFile(const std::string &)","cluster":"test_cluster_two_shards_localhost","shard":"2","rows":"1","bytes":"8"} +{"operation_name":"auto DB::DistributedSink::runWritingJob(DB::DistributedSink::JobReplica &, const DB::Block &, size_t)::(anonymous class)::operator()() const","cluster":"test_cluster_two_shards_localhost","shard":"1","rows":"1","bytes":"8"} +{"operation_name":"auto DB::DistributedSink::runWritingJob(DB::DistributedSink::JobReplica &, const DB::Block &, size_t)::(anonymous class)::operator()() const","cluster":"test_cluster_two_shards_localhost","shard":"2","rows":"1","bytes":"8"} +{"operation_name":"auto DB::DistributedSink::runWritingJob(DB::DistributedSink::JobReplica &, const DB::Block &, size_t)::(anonymous class)::operator()() const","cluster":"test_cluster_two_shards_localhost","shard":"1","rows":"1","bytes":"8"} +{"operation_name":"auto DB::DistributedSink::runWritingJob(DB::DistributedSink::JobReplica &, const DB::Block &, size_t)::(anonymous class)::operator()() const","cluster":"test_cluster_two_shards_localhost","shard":"2","rows":"1","bytes":"8"} diff --git a/tests/queries/0_stateless/02417_opentelemetry_insert_on_distributed_table.sh b/tests/queries/0_stateless/02417_opentelemetry_insert_on_distributed_table.sh new file mode 100755 index 00000000000..9ac5f061d4a --- /dev/null +++ b/tests/queries/0_stateless/02417_opentelemetry_insert_on_distributed_table.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, distributed + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + + +# This function takes 4 arguments: +# $1 - OpenTelemetry Trace Id +# $2 - value of insert_distributed_sync +# $3 - value of prefer_localhost_replica +# $4 - a String that helps to debug +function insert() +{ + echo "INSERT INTO ${CLICKHOUSE_DATABASE}.dist_opentelemetry SETTINGS insert_distributed_sync=$2, prefer_localhost_replica=$3 VALUES(1),(2)" | + ${CLICKHOUSE_CURL} \ + -X POST \ + -H "traceparent: 00-$1-5150000000000515-01" \ + -H "tracestate: $4" \ + "${CLICKHOUSE_URL}" \ + --data @- +} + +function check_span() +{ +${CLICKHOUSE_CLIENT} -nq " + SYSTEM FLUSH LOGS; + + SELECT operation_name, + attribute['clickhouse.cluster'] AS cluster, + attribute['clickhouse.shard_num'] AS shard, + attribute['clickhouse.rows'] AS rows, + attribute['clickhouse.bytes'] AS bytes + FROM system.opentelemetry_span_log + WHERE finish_date >= yesterday() + AND lower(hex(trace_id)) = '${1}' + AND attribute['clickhouse.distributed'] = '${CLICKHOUSE_DATABASE}.dist_opentelemetry' + AND attribute['clickhouse.remote'] = '${CLICKHOUSE_DATABASE}.local_opentelemetry' + ORDER BY attribute['clickhouse.shard_num'] + Format JSONEachRow + ;" +} + + +# +# Prepare tables for tests +# +${CLICKHOUSE_CLIENT} -nq " +DROP TABLE IF EXISTS ${CLICKHOUSE_DATABASE}.dist_opentelemetry; +DROP TABLE IF EXISTS ${CLICKHOUSE_DATABASE}.local_opentelemetry; + +CREATE TABLE ${CLICKHOUSE_DATABASE}.dist_opentelemetry (key UInt64) Engine=Distributed('test_cluster_two_shards_localhost', ${CLICKHOUSE_DATABASE}, local_opentelemetry, key % 2); +CREATE TABLE ${CLICKHOUSE_DATABASE}.local_opentelemetry (key UInt64) Engine=MergeTree ORDER BY key; +" + +# +# test1 +# +trace_id=$(${CLICKHOUSE_CLIENT} -q "select lower(hex(generateUUIDv4()))"); +insert $trace_id 0 1 "async-insert-writeToLocal" +check_span $trace_id + +# +# test2 +# +trace_id=$(${CLICKHOUSE_CLIENT} -q "select lower(hex(generateUUIDv4()))"); +insert $trace_id 0 0 "async-insert-writeToRemote" +check_span $trace_id + +# +# test3 +# +trace_id=$(${CLICKHOUSE_CLIENT} -q "select lower(hex(generateUUIDv4()))"); +insert $trace_id 1 1 "sync-insert-writeToLocal" +check_span $trace_id + +# +# test4 +# +trace_id=$(${CLICKHOUSE_CLIENT} -q "select lower(hex(generateUUIDv4()))"); +insert $trace_id 1 0 "sync-insert-writeToRemote" +check_span $trace_id + +# +# Cleanup +# +${CLICKHOUSE_CLIENT} -nq " +DROP TABLE ${CLICKHOUSE_DATABASE}.dist_opentelemetry; +DROP TABLE ${CLICKHOUSE_DATABASE}.local_opentelemetry; +" diff --git a/tests/queries/0_stateless/02418_do_not_return_empty_blocks_from_ConvertingAggregatedToChunksTransform.reference b/tests/queries/0_stateless/02418_do_not_return_empty_blocks_from_ConvertingAggregatedToChunksTransform.reference new file mode 100644 index 00000000000..f2586c9c42a --- /dev/null +++ b/tests/queries/0_stateless/02418_do_not_return_empty_blocks_from_ConvertingAggregatedToChunksTransform.reference @@ -0,0 +1,6 @@ +┌─number─┐ +│ 42 │ +└────────┘ +┌─number─┐ +│ 42 │ +└────────┘ diff --git a/tests/queries/0_stateless/02418_do_not_return_empty_blocks_from_ConvertingAggregatedToChunksTransform.sh b/tests/queries/0_stateless/02418_do_not_return_empty_blocks_from_ConvertingAggregatedToChunksTransform.sh new file mode 100755 index 00000000000..08c7e18e12c --- /dev/null +++ b/tests/queries/0_stateless/02418_do_not_return_empty_blocks_from_ConvertingAggregatedToChunksTransform.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +set -ue + +unset CLICKHOUSE_LOG_COMMENT + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +${CLICKHOUSE_CURL} \ + $CLICKHOUSE_URL \ + --get \ + --data-urlencode "query= + select number + from numbers_mt(1e6) + where number = 42 + group by number + settings max_threads = 10, max_bytes_before_external_group_by = 1, group_by_two_level_threshold = 1 + format PrettyCompact" + +${CLICKHOUSE_CURL} \ + $CLICKHOUSE_URL \ + --get \ + --data-urlencode "query= + select number + from numbers_mt(1e6) + where number = 42 + group by number + settings max_threads = 10, max_bytes_before_external_group_by = 0, group_by_two_level_threshold = 1 + format PrettyCompact" diff --git a/tests/queries/0_stateless/02418_keeper_map_keys_limit.reference b/tests/queries/0_stateless/02418_keeper_map_keys_limit.reference new file mode 100644 index 00000000000..95c45d6da51 --- /dev/null +++ b/tests/queries/0_stateless/02418_keeper_map_keys_limit.reference @@ -0,0 +1,4 @@ +2 +3 +4 +4 diff --git a/tests/queries/0_stateless/02418_keeper_map_keys_limit.sql b/tests/queries/0_stateless/02418_keeper_map_keys_limit.sql new file mode 100644 index 00000000000..3d2055b85ea --- /dev/null +++ b/tests/queries/0_stateless/02418_keeper_map_keys_limit.sql @@ -0,0 +1,23 @@ +-- Tags: no-ordinary-database, no-fasttest + +DROP TABLE IF EXISTS 02418_test SYNC; + +CREATE TABLE 02418_test (key UInt64, value Float64) Engine=KeeperMap('/' || currentDatabase() || '/test2418', 3) PRIMARY KEY(key); + +INSERT INTO 02418_test VALUES (1, 1.1), (2, 2.2); +SELECT count() FROM 02418_test; + +INSERT INTO 02418_test VALUES (3, 3.3), (4, 4.4); -- { serverError 290 } + +INSERT INTO 02418_test VALUES (1, 2.1), (2, 3.2), (3, 3.3); +SELECT count() FROM 02418_test; + +CREATE TABLE 02418_test_another (key UInt64, value Float64) Engine=KeeperMap('/' || currentDatabase() || '/test2418', 4) PRIMARY KEY(key); +INSERT INTO 02418_test VALUES (4, 4.4); -- { serverError 290 } +INSERT INTO 02418_test_another VALUES (4, 4.4); + +SELECT count() FROM 02418_test; +SELECT count() FROM 02418_test_another; + +DROP TABLE 02418_test SYNC; +DROP TABLE 02418_test_another SYNC; diff --git a/tests/queries/0_stateless/02419_keeper_map_primary_key.reference b/tests/queries/0_stateless/02419_keeper_map_primary_key.reference new file mode 100644 index 00000000000..8394d9f34a7 --- /dev/null +++ b/tests/queries/0_stateless/02419_keeper_map_primary_key.reference @@ -0,0 +1,6 @@ +1.1 +2.2 +1.1 +2.2 +1.1 +2.2 diff --git a/tests/queries/0_stateless/02419_keeper_map_primary_key.sh b/tests/queries/0_stateless/02419_keeper_map_primary_key.sh new file mode 100755 index 00000000000..c43c5bb6408 --- /dev/null +++ b/tests/queries/0_stateless/02419_keeper_map_primary_key.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# Tags: no-ordinary-database, no-fasttest, long + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS 02419_test SYNC;" + +test_primary_key() +{ + $CLICKHOUSE_CLIENT -nm -q " + CREATE TABLE 02419_test (key UInt64, value Float64) Engine=KeeperMap('/' || currentDatabase() || '/test2418', 3) PRIMARY KEY($1); + INSERT INTO 02419_test VALUES (1, 1.1), (2, 2.2); + SELECT value FROM 02419_test WHERE key = 1; + SELECT value FROM 02419_test WHERE key IN (2, 3); + DROP TABLE 02419_test SYNC; + " +} + +test_primary_key "sipHash64(key + 42) * 12212121212121" +test_primary_key "reverse(concat(CAST(key, 'String'), 'some string'))" +test_primary_key "hex(toFloat32(key))" diff --git a/tests/queries/0_stateless/02420_key_condition_actions_dag_bug_40599.reference b/tests/queries/0_stateless/02420_key_condition_actions_dag_bug_40599.reference new file mode 100644 index 00000000000..8bd1af11bf2 --- /dev/null +++ b/tests/queries/0_stateless/02420_key_condition_actions_dag_bug_40599.reference @@ -0,0 +1 @@ +2000 diff --git a/tests/queries/0_stateless/02420_key_condition_actions_dag_bug_40599.sql b/tests/queries/0_stateless/02420_key_condition_actions_dag_bug_40599.sql new file mode 100644 index 00000000000..4d2feacfde7 --- /dev/null +++ b/tests/queries/0_stateless/02420_key_condition_actions_dag_bug_40599.sql @@ -0,0 +1,10 @@ +create table tba (event_id Int64, event_dt Int64) Engine =MergeTree order by event_id ; +insert into tba select number%500, 20220822 from numbers(1e6); + +select count() from ( + SELECT event_dt FROM ( + select event_dt, 403 AS event_id from ( + select event_dt from tba as tba + where event_id = 9 and ((tba.event_dt >= 20220822 and tba.event_dt <= 20220822)) + ) + ) tba WHERE tba.event_dt >= 20220822 and tba.event_dt <= 20220822 and event_id = 403 ); diff --git a/tests/queries/0_stateless/02421_formats_with_totals_and_extremes.reference b/tests/queries/0_stateless/02421_formats_with_totals_and_extremes.reference new file mode 100644 index 00000000000..ee8e589089c Binary files /dev/null and b/tests/queries/0_stateless/02421_formats_with_totals_and_extremes.reference differ diff --git a/tests/queries/0_stateless/02421_formats_with_totals_and_extremes.sql.j2 b/tests/queries/0_stateless/02421_formats_with_totals_and_extremes.sql.j2 new file mode 100644 index 00000000000..32738766199 --- /dev/null +++ b/tests/queries/0_stateless/02421_formats_with_totals_and_extremes.sql.j2 @@ -0,0 +1,22 @@ +-- Tags: no-fasttest + +set output_format_write_statistics=0; + +{% for format in ['CSV', 'TSV', 'XML', 'Vertical', 'Pretty', 'JSON', 'JSONCompact'] -%} + +select '{{ format }}'; +select sum(number) from numbers(10) group by number % 2 with totals format {{ format }} settings extremes=1; +select ''; + +{% endfor -%} + +select 'Formats without totals and extremes:'; + +{% for format in ['CustomSeparated', 'JSONEachRow', 'JSONCompactEachRow', 'RowBinary', 'MsgPack', 'Markdown', 'SQLInsert', 'Values', 'TSKV'] -%} + +select '{{ format }}'; +select sum(number) from numbers(10) group by number % 2 with totals format {{ format }} settings extremes=1; +select ''; + +{% endfor -%} + diff --git a/tests/queries/0_stateless/02421_record_errors_row_by_input_format.reference b/tests/queries/0_stateless/02421_record_errors_row_by_input_format.reference new file mode 100644 index 00000000000..67ec09b70b7 --- /dev/null +++ b/tests/queries/0_stateless/02421_record_errors_row_by_input_format.reference @@ -0,0 +1,6 @@ +default data 2 Row 2:\nColumn 0, name: c1, type: UInt8, parsed text: "2"\nColumn 1, name: c2, type: UInt8, ERROR: text "ab,34,4" is not like UInt8 2,a +default data 3 Row 3:\nColumn 0, name: c1, type: UInt8, ERROR: text "b,34,45," is not like UInt8 b,3 +default data 5 Row 5:\nColumn 0, name: c1, type: UInt8, parsed text: "5"\nColumn 1, name: c2, type: UInt8, ERROR: text "c6,6" is not like UInt8 5,c +\N data 2 Row 2:\nColumn 0, name: A, type: UInt8, parsed text: "2"\nColumn 1, name: B, type: UInt8, ERROR: text "ab,34,4" is not like UInt8 2,a +\N data 3 Row 3:\nColumn 0, name: A, type: UInt8, ERROR: text "b,34,45," is not like UInt8 b,3 +\N data 5 Row 5:\nColumn 0, name: A, type: UInt8, parsed text: "5"\nColumn 1, name: B, type: UInt8, ERROR: text "c6,6" is not like UInt8 5,c diff --git a/tests/queries/0_stateless/02421_record_errors_row_by_input_format.sh b/tests/queries/0_stateless/02421_record_errors_row_by_input_format.sh new file mode 100755 index 00000000000..dda61512936 --- /dev/null +++ b/tests/queries/0_stateless/02421_record_errors_row_by_input_format.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# Tags: no-parallel, no-fasttest + +set -eu + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +# Data preparation. + +CLICKHOUSE_USER_FILES_PATH=$(clickhouse-client --query "select _path, _file from file('nonexist.txt', 'CSV', 'val1 char')" 2>&1 | grep Exception | awk '{gsub("/nonexist.txt","",$9); print $9}') + +mkdir -p ${CLICKHOUSE_USER_FILES_PATH}/ +echo -e "1,1\n2,a\nb,3\n4,4\n5,c\n6,6" > ${CLICKHOUSE_USER_FILES_PATH}/a.csv + +${CLICKHOUSE_CLIENT} --query "drop table if exists data;" +${CLICKHOUSE_CLIENT} --query "create table data (A UInt8, B UInt8) engine=MergeTree() order by A;" + +# Server side +${CLICKHOUSE_CLIENT} --input_format_allow_errors_num 4 --input_format_record_errors_file_path "errors_server" --query "insert into data select * from file('a.csv', 'CSV', 'c1 UInt8, c2 UInt8');" +sleep 2 +${CLICKHOUSE_CLIENT} --query "select * except (time) from file('errors_server', 'CSV', 'time DateTime, database Nullable(String), table Nullable(String), offset UInt32, reason String, raw_data String');" + +# Client side +${CLICKHOUSE_CLIENT} --input_format_allow_errors_num 4 --input_format_record_errors_file_path "${CLICKHOUSE_USER_FILES_PATH}/errors_client" --query "insert into data(A, B) format CSV" < ${CLICKHOUSE_USER_FILES_PATH}/a.csv +sleep 2 +${CLICKHOUSE_CLIENT} --query "select * except (time) from file('errors_client', 'CSV', 'time DateTime, database Nullable(String), table Nullable(String), offset UInt32, reason String, raw_data String');" + +# Restore +${CLICKHOUSE_CLIENT} --query "drop table if exists data;" +rm ${CLICKHOUSE_USER_FILES_PATH}/a.csv +rm ${CLICKHOUSE_USER_FILES_PATH}/errors_server +rm ${CLICKHOUSE_USER_FILES_PATH}/errors_client + diff --git a/tests/queries/0_stateless/02421_simple_queries_for_opentelemetry.reference b/tests/queries/0_stateless/02421_simple_queries_for_opentelemetry.reference new file mode 100644 index 00000000000..d167d905636 --- /dev/null +++ b/tests/queries/0_stateless/02421_simple_queries_for_opentelemetry.reference @@ -0,0 +1,4 @@ +{"query":"show processlist format Null\n "} +{"query":"show databases format Null\n "} +{"query":"insert into opentelemetry_test values","read_rows":"3","written_rows":"3"} +{"query":"select * from opentelemetry_test format Null\n ","read_rows":"3","written_rows":""} diff --git a/tests/queries/0_stateless/02421_simple_queries_for_opentelemetry.sh b/tests/queries/0_stateless/02421_simple_queries_for_opentelemetry.sh new file mode 100755 index 00000000000..98b571c5968 --- /dev/null +++ b/tests/queries/0_stateless/02421_simple_queries_for_opentelemetry.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +# This function takes 2 arguments: +# $1 - query id +# $2 - query +function execute_query() +{ + ${CLICKHOUSE_CLIENT} --opentelemetry_start_trace_probability=1 --query_id $1 -nq " + ${2} + " +} + +# For some queries, it's not possible to know how many bytes/rows are read when tests are executed on CI, +# so we only to check the db.statement only +function check_query_span_query_only() +{ +${CLICKHOUSE_CLIENT} -nq " + SYSTEM FLUSH LOGS; + SELECT attribute['db.statement'] as query + FROM system.opentelemetry_span_log + WHERE finish_date >= yesterday() + AND operation_name = 'query' + AND attribute['clickhouse.query_id'] = '${1}' + Format JSONEachRow + ;" +} + +function check_query_span() +{ +${CLICKHOUSE_CLIENT} -nq " + SYSTEM FLUSH LOGS; + SELECT attribute['db.statement'] as query, + attribute['clickhouse.read_rows'] as read_rows, + attribute['clickhouse.written_rows'] as written_rows + FROM system.opentelemetry_span_log + WHERE finish_date >= yesterday() + AND operation_name = 'query' + AND attribute['clickhouse.query_id'] = '${1}' + Format JSONEachRow + ;" +} + +# +# Set up +# +${CLICKHOUSE_CLIENT} -nq " +DROP TABLE IF EXISTS ${CLICKHOUSE_DATABASE}.opentelemetry_test; +CREATE TABLE ${CLICKHOUSE_DATABASE}.opentelemetry_test (id UInt64) Engine=MergeTree Order By id; +" + +# test 1, a query that has special path in the code +# Format Null is used to make sure no output is generated so that it won't pollute the reference file +query_id=$(${CLICKHOUSE_CLIENT} -q "select generateUUIDv4()"); +execute_query $query_id 'show processlist format Null' +check_query_span_query_only "$query_id" + +# test 2, a normal show command +query_id=$(${CLICKHOUSE_CLIENT} -q "select generateUUIDv4()"); +execute_query $query_id 'show databases format Null' +check_query_span_query_only "$query_id" + +# test 3, a normal insert query on local table +query_id=$(${CLICKHOUSE_CLIENT} -q "select generateUUIDv4()"); +execute_query $query_id 'insert into opentelemetry_test values(1)(2)(3)' +check_query_span "$query_id" + +# test 4, a normal select query +query_id=$(${CLICKHOUSE_CLIENT} -q "select generateUUIDv4()"); +execute_query $query_id 'select * from opentelemetry_test format Null' +check_query_span $query_id + + +# +# Tear down +# +${CLICKHOUSE_CLIENT} -q " +DROP TABLE IF EXISTS ${CLICKHOUSE_DATABASE}.opentelemetry_test; +" \ No newline at end of file diff --git a/tests/queries/0_stateless/02421_type_json_async_insert.reference b/tests/queries/0_stateless/02421_type_json_async_insert.reference new file mode 100644 index 00000000000..f3d96ebf2d0 --- /dev/null +++ b/tests/queries/0_stateless/02421_type_json_async_insert.reference @@ -0,0 +1,5 @@ +Cannot parse object +0 +0 +Cannot parse object +aaa diff --git a/tests/queries/0_stateless/02421_type_json_async_insert.sh b/tests/queries/0_stateless/02421_type_json_async_insert.sh new file mode 100755 index 00000000000..8aa0d510dbb --- /dev/null +++ b/tests/queries/0_stateless/02421_type_json_async_insert.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS t_json_async_insert" +$CLICKHOUSE_CLIENT --allow_experimental_object_type=1 -q "CREATE TABLE t_json_async_insert (data JSON) ENGINE = MergeTree ORDER BY tuple()" + +$CLICKHOUSE_CLIENT --async_insert=1 --wait_for_async_insert=1 -q 'INSERT INTO t_json_async_insert FORMAT JSONAsObject {"aaa"}' 2>&1 | grep -o -m1 "Cannot parse object" +$CLICKHOUSE_CLIENT -q "SELECT count() FROM t_json_async_insert" +$CLICKHOUSE_CLIENT -q "SELECT count() FROM system.parts WHERE database = '$CLICKHOUSE_DATABASE' AND table = 't_json_async_insert'" + +$CLICKHOUSE_CLIENT --async_insert=1 --wait_for_async_insert=1 -q 'INSERT INTO t_json_async_insert FORMAT JSONAsObject {"aaa"}' 2>&1 | grep -o -m1 "Cannot parse object" & +$CLICKHOUSE_CLIENT --async_insert=1 --wait_for_async_insert=1 -q 'INSERT INTO t_json_async_insert FORMAT JSONAsObject {"k1": "aaa"}' & + +wait + +$CLICKHOUSE_CLIENT -q "SELECT data.k1 FROM t_json_async_insert ORDER BY data.k1" +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS t_json_async_insert" diff --git a/tests/queries/0_stateless/02421_type_json_empty_parts.reference b/tests/queries/0_stateless/02421_type_json_empty_parts.reference new file mode 100644 index 00000000000..f360b4b92cd --- /dev/null +++ b/tests/queries/0_stateless/02421_type_json_empty_parts.reference @@ -0,0 +1,26 @@ +Collapsing +0 +0 +id UInt64 +s Int8 +data Tuple(_dummy UInt8) +DELETE all +2 +1 +id UInt64 +data Tuple(k1 String, k2 String) +0 +0 +id UInt64 +data Tuple(_dummy UInt8) +TTL +1 +1 +id UInt64 +d Date +data Tuple(k1 String, k2 String) +0 +0 +id UInt64 +d Date +data Tuple(_dummy UInt8) diff --git a/tests/queries/0_stateless/02421_type_json_empty_parts.sql b/tests/queries/0_stateless/02421_type_json_empty_parts.sql new file mode 100644 index 00000000000..409a2b18a49 --- /dev/null +++ b/tests/queries/0_stateless/02421_type_json_empty_parts.sql @@ -0,0 +1,61 @@ +-- Tags: no-fasttest + +SET allow_experimental_object_type = 1; + +DROP TABLE IF EXISTS t_json_empty_parts; + +SELECT 'Collapsing'; +CREATE TABLE t_json_empty_parts (id UInt64, s Int8, data JSON) ENGINE = CollapsingMergeTree(s) ORDER BY id; + +INSERT INTO t_json_empty_parts VALUES (1, 1, '{"k1": "aaa"}') (1, -1, '{"k2": "bbb"}'); + +SELECT count() FROM t_json_empty_parts; +SELECT count() FROM system.parts WHERE table = 't_json_empty_parts' AND database = currentDatabase() AND active; +DESC TABLE t_json_empty_parts SETTINGS describe_extend_object_types = 1; + +DROP TABLE t_json_empty_parts; + +DROP TABLE IF EXISTS t_json_empty_parts; + +SELECT 'DELETE all'; +CREATE TABLE t_json_empty_parts (id UInt64, data JSON) ENGINE = MergeTree ORDER BY id; + +INSERT INTO t_json_empty_parts VALUES (1, '{"k1": "aaa"}') (2, '{"k2": "bbb"}'); + +SELECT count() FROM t_json_empty_parts; +SELECT count() FROM system.parts WHERE table = 't_json_empty_parts' AND database = currentDatabase() AND active; +DESC TABLE t_json_empty_parts SETTINGS describe_extend_object_types = 1; + +SET mutations_sync = 2; +ALTER TABLE t_json_empty_parts DELETE WHERE 1; + +DETACH TABLE t_json_empty_parts; +ATTACH TABLE t_json_empty_parts; + +SELECT count() FROM t_json_empty_parts; +SELECT count() FROM system.parts WHERE table = 't_json_empty_parts' AND database = currentDatabase() AND active; +DESC TABLE t_json_empty_parts SETTINGS describe_extend_object_types = 1; + +DROP TABLE IF EXISTS t_json_empty_parts; + +SELECT 'TTL'; +CREATE TABLE t_json_empty_parts (id UInt64, d Date, data JSON) ENGINE = MergeTree ORDER BY id TTL d WHERE id % 2 = 1; + +INSERT INTO t_json_empty_parts VALUES (1, '2000-01-01', '{"k1": "aaa"}') (2, '2000-01-01', '{"k2": "bbb"}'); +OPTIMIZE TABLE t_json_empty_parts FINAL; + +SELECT count() FROM t_json_empty_parts; +SELECT count() FROM system.parts WHERE table = 't_json_empty_parts' AND database = currentDatabase() AND active; +DESC TABLE t_json_empty_parts SETTINGS describe_extend_object_types = 1; + +ALTER TABLE t_json_empty_parts MODIFY TTL d; +OPTIMIZE TABLE t_json_empty_parts FINAL; + +DETACH TABLE t_json_empty_parts; +ATTACH TABLE t_json_empty_parts; + +SELECT count() FROM t_json_empty_parts; +SELECT count() FROM system.parts WHERE table = 't_json_empty_parts' AND database = currentDatabase() AND active; +DESC TABLE t_json_empty_parts SETTINGS describe_extend_object_types = 1; + +DROP TABLE IF EXISTS t_json_empty_parts; diff --git a/tests/integration/test_catboost_model_reload/__init__.py b/tests/queries/0_stateless/02422_msgpack_uuid_wrong_column.reference similarity index 100% rename from tests/integration/test_catboost_model_reload/__init__.py rename to tests/queries/0_stateless/02422_msgpack_uuid_wrong_column.reference diff --git a/tests/queries/0_stateless/02422_msgpack_uuid_wrong_column.sql b/tests/queries/0_stateless/02422_msgpack_uuid_wrong_column.sql new file mode 100644 index 00000000000..4d790354d51 --- /dev/null +++ b/tests/queries/0_stateless/02422_msgpack_uuid_wrong_column.sql @@ -0,0 +1,4 @@ +-- Tags: no-parallel, no-fasttest + +insert into function file(02422_data.msgpack) select toUUID('f4cdd80d-5d15-4bdc-9527-adcca635ec1f') as uuid settings output_format_msgpack_uuid_representation='ext'; +select * from file(02422_data.msgpack, auto, 'x Int32'); -- {serverError ILLEGAL_COLUMN} diff --git a/tests/queries/0_stateless/02423_insert_stats_behaviour.reference b/tests/queries/0_stateless/02423_insert_stats_behaviour.reference new file mode 100644 index 00000000000..6aabb57a8bd --- /dev/null +++ b/tests/queries/0_stateless/02423_insert_stats_behaviour.reference @@ -0,0 +1,12 @@ +read_rows=1 read_bytes=8 written_rows=1 written_bytes=8 query=INSERT into target_1 FORMAT CSV +read_rows=10 read_bytes=80 written_rows=10 written_bytes=80 query=INSERT INTO target_1 FORMAT Native\n +read_rows=10 read_bytes=80 written_rows=10 written_bytes=80 query=INSERT INTO target_1 FORMAT RowBinary\n +read_rows=5 read_bytes=40 written_rows=4 written_bytes=32 query=INSERT into floats FORMAT CSV +read_rows=32 read_bytes=256 written_rows=40 written_bytes=320 query=INSERT INTO floats FORMAT Native\n +read_rows=32 read_bytes=256 written_rows=40 written_bytes=320 query=INSERT INTO floats FORMAT RowBinary\n +read_rows=1 read_bytes=8 written_rows=1 written_bytes=8 source_query=INSERT into floats FORMAT CSV view_query=SELECT * FROM default.floats +read_rows=3 read_bytes=24 written_rows=2 written_bytes=16 source_query=INSERT into floats FORMAT CSV view_query=SELECT * FROM default.floats, numbers(2) AS n +read_rows=10 read_bytes=80 written_rows=10 written_bytes=80 source_query=INSERT INTO floats FORMAT Native\n view_query=SELECT * FROM default.floats +read_rows=12 read_bytes=96 written_rows=20 written_bytes=160 source_query=INSERT INTO floats FORMAT Native\n view_query=SELECT * FROM default.floats, numbers(2) AS n +read_rows=10 read_bytes=80 written_rows=10 written_bytes=80 source_query=INSERT INTO floats FORMAT RowBinary\n view_query=SELECT * FROM default.floats +read_rows=12 read_bytes=96 written_rows=20 written_bytes=160 source_query=INSERT INTO floats FORMAT RowBinary\n view_query=SELECT * FROM default.floats, numbers(2) AS n diff --git a/tests/queries/0_stateless/02423_insert_stats_behaviour.sh b/tests/queries/0_stateless/02423_insert_stats_behaviour.sh new file mode 100755 index 00000000000..b85ca311101 --- /dev/null +++ b/tests/queries/0_stateless/02423_insert_stats_behaviour.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +$CLICKHOUSE_CLIENT -q "CREATE TABLE floats (v Float64) Engine=MergeTree() ORDER BY tuple();" +$CLICKHOUSE_CLIENT -q "CREATE TABLE target_1 (v Float64) Engine=MergeTree() ORDER BY tuple();" +$CLICKHOUSE_CLIENT -q "CREATE TABLE target_2 (v Float64) Engine=MergeTree() ORDER BY tuple();" +$CLICKHOUSE_CLIENT -q "CREATE MATERIALIZED VIEW floats_to_target TO target_1 AS SELECT * FROM floats" +$CLICKHOUSE_CLIENT -q "CREATE MATERIALIZED VIEW floats_to_target_2 TO target_2 AS SELECT * FROM floats, numbers(2) n" + +# Insertions into table without MVs +$CLICKHOUSE_CLIENT -q "INSERT into target_1 FORMAT CSV 1.0" +$CLICKHOUSE_LOCAL -q "SELECT number::Float64 AS v FROM numbers(10)" --format Native | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&query=INSERT+INTO+target_1+FORMAT+Native" --data-binary @- +$CLICKHOUSE_LOCAL -q "SELECT number::Float64 AS v FROM numbers(10)" --format RowBinary | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&query=INSERT+INTO+target_1+FORMAT+RowBinary" --data-binary @- + +# Insertions into table without 2 MVs (1:1 and 1:2 rows) +$CLICKHOUSE_CLIENT -q "INSERT into floats FORMAT CSV 1.0" +$CLICKHOUSE_LOCAL -q "SELECT number::Float64 AS v FROM numbers(10)" --format Native | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&query=INSERT+INTO+floats+FORMAT+Native" --data-binary @- +$CLICKHOUSE_LOCAL -q "SELECT number::Float64 AS v FROM numbers(10)" --format RowBinary | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&query=INSERT+INTO+floats+FORMAT+RowBinary" --data-binary @- + +$CLICKHOUSE_CLIENT -q "SYSTEM FLUSH LOGS" +$CLICKHOUSE_CLIENT -q \ + "SELECT + read_rows, + read_bytes, + written_rows, + written_bytes, + query + FROM system.query_log + WHERE + event_date >= yesterday() + AND event_time > now() - INTERVAL 600 SECOND + AND type = 'QueryFinish' + AND query_kind = 'Insert' + AND current_database == currentDatabase() + ORDER BY event_time_microseconds ASC + FORMAT TSKV" + +$CLICKHOUSE_CLIENT -q \ + "SELECT + read_rows, + read_bytes, + written_rows, + written_bytes, + ql.query as source_query, + view_query + FROM system.query_views_log + INNER JOIN + ( + SELECT + query_id, query, event_time_microseconds + FROM system.query_log + WHERE + event_date >= yesterday() + AND event_time > now() - INTERVAL 600 SECOND + AND type = 'QueryFinish' + AND query_kind = 'Insert' + AND current_database == currentDatabase() + ) ql + ON system.query_views_log.initial_query_id = ql.query_id + ORDER BY ql.event_time_microseconds ASC, view_query ASC + FORMAT TSKV" diff --git a/tests/queries/0_stateless/02413_model_evaluate_smoke.reference b/tests/queries/0_stateless/02423_multidimensional_array_get_data_at.reference similarity index 100% rename from tests/queries/0_stateless/02413_model_evaluate_smoke.reference rename to tests/queries/0_stateless/02423_multidimensional_array_get_data_at.reference diff --git a/tests/queries/0_stateless/02423_multidimensional_array_get_data_at.sql b/tests/queries/0_stateless/02423_multidimensional_array_get_data_at.sql new file mode 100644 index 00000000000..a47fbdfc789 --- /dev/null +++ b/tests/queries/0_stateless/02423_multidimensional_array_get_data_at.sql @@ -0,0 +1,7 @@ +SELECT formatRow('RawBLOB', [[[33]], []]); -- { serverError 48 } +SELECT formatRow('RawBLOB', [[[]], []]); -- { serverError 48 } +SELECT formatRow('RawBLOB', [[[[[[[0x48, 0x65, 0x6c, 0x6c, 0x6f]]]]]], []]); -- { serverError 48 } +SELECT formatRow('RawBLOB', []::Array(Array(Nothing))); -- { serverError 48 } +SELECT formatRow('RawBLOB', [[], [['Hello']]]); -- { serverError 48 } +SELECT formatRow('RawBLOB', [[['World']], []]); -- { serverError 48 } +SELECT formatRow('RawBLOB', []::Array(String)); -- { serverError 48 } diff --git a/tests/queries/0_stateless/02424_pod_array_overflow.reference b/tests/queries/0_stateless/02424_pod_array_overflow.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02424_pod_array_overflow.sql b/tests/queries/0_stateless/02424_pod_array_overflow.sql new file mode 100644 index 00000000000..4b85d5be029 --- /dev/null +++ b/tests/queries/0_stateless/02424_pod_array_overflow.sql @@ -0,0 +1 @@ +SELECT * FROM format(Native, '\x02\x02\x02\x6b\x30\x1a\x4d\x61\x70\x28\x46\x69\x78\x65\x64\x53\x74\x72\x69\x6e\x67\x28\x31\x29\x2c\x20\x49\x6e\x74\x36\x34\x29\x01\x00\x00\x00\x00\x00\x00\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x7f\x00\x7f\xff\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x64\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xcf\x31\x3f\x56\x69\x11\x89\x25'); -- { serverError 128 } diff --git a/tests/queries/0_stateless/02425_categorical_information_value_properties.reference b/tests/queries/0_stateless/02425_categorical_information_value_properties.reference new file mode 100644 index 00000000000..bc3af98b060 --- /dev/null +++ b/tests/queries/0_stateless/02425_categorical_information_value_properties.reference @@ -0,0 +1,12 @@ +0.347 +0.5 +0.347 +0.347 +[nan] +[nan] +[nan] +[nan] +[0] +\N +[nan] +[0,0] diff --git a/tests/queries/0_stateless/02425_categorical_information_value_properties.sql b/tests/queries/0_stateless/02425_categorical_information_value_properties.sql new file mode 100644 index 00000000000..81ed8400680 --- /dev/null +++ b/tests/queries/0_stateless/02425_categorical_information_value_properties.sql @@ -0,0 +1,14 @@ +SELECT round(arrayJoin(categoricalInformationValue(x.1, x.2)), 3) FROM (SELECT arrayJoin([(0, 0), (NULL, 2), (1, 0), (1, 1)]) AS x); +SELECT corr(c1, c2) FROM VALUES((0, 0), (NULL, 2), (1, 0), (1, 1)); +SELECT round(arrayJoin(categoricalInformationValue(c1, c2)), 3) FROM VALUES((0, 0), (NULL, 2), (1, 0), (1, 1)); +SELECT round(arrayJoin(categoricalInformationValue(c1, c2)), 3) FROM VALUES((0, 0), (NULL, 1), (1, 0), (1, 1)); +SELECT categoricalInformationValue(c1, c2) FROM VALUES((0, 0), (NULL, 1)); +SELECT categoricalInformationValue(c1, c2) FROM VALUES((NULL, 1)); -- { serverError 43 } +SELECT categoricalInformationValue(dummy, dummy); +SELECT categoricalInformationValue(dummy, dummy) WHERE 0; +SELECT categoricalInformationValue(c1, c2) FROM VALUES((toNullable(0), 0)); +SELECT groupUniqArray(*) FROM VALUES(toNullable(0)); +SELECT groupUniqArray(*) FROM VALUES(NULL); +SELECT categoricalInformationValue(c1, c2) FROM VALUES((NULL, NULL)); -- { serverError 43 } +SELECT categoricalInformationValue(c1, c2) FROM VALUES((0, 0), (NULL, 0)); +SELECT quantiles(0.5, 0.9)(c1) FROM VALUES(0::Nullable(UInt8)); diff --git a/tests/queries/0_stateless/02426_low_cardinality_fixed_string_insert_field.reference b/tests/queries/0_stateless/02426_low_cardinality_fixed_string_insert_field.reference new file mode 100644 index 00000000000..3bfced8d8bd --- /dev/null +++ b/tests/queries/0_stateless/02426_low_cardinality_fixed_string_insert_field.reference @@ -0,0 +1 @@ +4908278 diff --git a/tests/queries/0_stateless/02426_low_cardinality_fixed_string_insert_field.sh b/tests/queries/0_stateless/02426_low_cardinality_fixed_string_insert_field.sh new file mode 100755 index 00000000000..dc9f1ec8ed2 --- /dev/null +++ b/tests/queries/0_stateless/02426_low_cardinality_fixed_string_insert_field.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +${CLICKHOUSE_LOCAL} --structure 'x LowCardinality(FixedString(2454139))' --input-format Values --output-format TSV --query "SELECT * FROM table" <<< '(1)' | wc -c diff --git a/tests/queries/0_stateless/02426_pod_array_overflow_2.reference b/tests/queries/0_stateless/02426_pod_array_overflow_2.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02426_pod_array_overflow_2.sql b/tests/queries/0_stateless/02426_pod_array_overflow_2.sql new file mode 100644 index 00000000000..52a00730227 --- /dev/null +++ b/tests/queries/0_stateless/02426_pod_array_overflow_2.sql @@ -0,0 +1 @@ +SELECT * FROM format(Native, 'k0\x23Array(Tuple(FixedString(1), Int64))\0\0\0\0\0\0\0�����\0����������������\0�\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0d\0\0\0\0\0\0\0\0\0\0\0\0\0�1?Vi�%'); -- { serverError 128 } diff --git a/tests/queries/0_stateless/02426_pod_array_overflow_3.reference b/tests/queries/0_stateless/02426_pod_array_overflow_3.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02426_pod_array_overflow_3.sql b/tests/queries/0_stateless/02426_pod_array_overflow_3.sql new file mode 100644 index 00000000000..857ba2ca28e --- /dev/null +++ b/tests/queries/0_stateless/02426_pod_array_overflow_3.sql @@ -0,0 +1 @@ +SELECT * FROM format(Native, '\x01\x01\x01x\x0CArray(UInt8)\x01\x00\xBD\xEF\xBF\xBD\xEF\xBF\xBD\xEF'); -- { serverError 128 } diff --git a/tests/queries/0_stateless/02427_column_nullable_ubsan.reference b/tests/queries/0_stateless/02427_column_nullable_ubsan.reference new file mode 100644 index 00000000000..e6c99ff9291 --- /dev/null +++ b/tests/queries/0_stateless/02427_column_nullable_ubsan.reference @@ -0,0 +1,10 @@ +0 999999 999999 +0 999998 999998 +0 999997 999997 +0 999996 999996 +0 999995 999995 +0 999994 999994 +0 999993 999993 +0 999992 999992 +0 999991 999991 +0 999990 999990 diff --git a/tests/queries/0_stateless/02427_column_nullable_ubsan.sql b/tests/queries/0_stateless/02427_column_nullable_ubsan.sql new file mode 100644 index 00000000000..3d1a51804a7 --- /dev/null +++ b/tests/queries/0_stateless/02427_column_nullable_ubsan.sql @@ -0,0 +1 @@ +SELECT * FROM (SELECT * FROM (SELECT 0 AS a, toNullable(number) AS b, toString(number) AS c FROM numbers(1000000.)) ORDER BY a DESC, b DESC, c ASC LIMIT 1500) LIMIT 10; diff --git a/tests/queries/0_stateless/02427_msan_group_array_resample.reference b/tests/queries/0_stateless/02427_msan_group_array_resample.reference new file mode 100644 index 00000000000..1a7f33e1d69 --- /dev/null +++ b/tests/queries/0_stateless/02427_msan_group_array_resample.reference @@ -0,0 +1 @@ +[[10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]] diff --git a/tests/queries/0_stateless/02427_msan_group_array_resample.sql b/tests/queries/0_stateless/02427_msan_group_array_resample.sql new file mode 100644 index 00000000000..6eccf59a6af --- /dev/null +++ b/tests/queries/0_stateless/02427_msan_group_array_resample.sql @@ -0,0 +1 @@ +SELECT arrayMap(x -> finalizeAggregation(x), state) FROM (SELECT groupArrayResample(9223372036854775806, 1048575, 65537)(number, number % 3), groupArrayStateResample(10, 2147483648, 65535)(number, number % 9223372036854775806) AS state FROM numbers(100)); diff --git a/utils/changelog/README.md b/utils/changelog/README.md index 8218af83d96..739229b49c9 100644 --- a/utils/changelog/README.md +++ b/utils/changelog/README.md @@ -13,6 +13,8 @@ python3 changelog.py -h Usage example: ``` +git fetch --tags # changelog.py depends on having the tags available, this will fetch them + python3 changelog.py --output=changelog-v22.4.1.2305-prestable.md --gh-user-or-token="$GITHUB_TOKEN" v21.6.2.7-prestable python3 changelog.py --output=changelog-v22.4.1.2305-prestable.md --gh-user-or-token="$USER" --gh-password="$PASSWORD" v21.6.2.7-prestable ``` diff --git a/utils/list-versions/update-docker-version.sh b/utils/list-versions/update-docker-version.sh new file mode 100755 index 00000000000..429da330a9f --- /dev/null +++ b/utils/list-versions/update-docker-version.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +set -e + +# We check only our code, that's why we skip contrib +GIT_ROOT=$(git rev-parse --show-cdup) +GIT_ROOT=${GIT_ROOT:-.} +VERSION=$(sed -e '1 s/^v//; 1 s/-.*//p; d' "$GIT_ROOT"/utils/list-versions/version_date.tsv) + +find "$GIT_ROOT/docker/server/" -name 'Dockerfile.*' -print0 | xargs -0 sed -i "/^ARG VERSION=/ s/^.*$/ARG VERSION=\"$VERSION\"/" diff --git a/utils/list-versions/version_date.tsv b/utils/list-versions/version_date.tsv index f2c8cfc4c76..f7df0345842 100644 --- a/utils/list-versions/version_date.tsv +++ b/utils/list-versions/version_date.tsv @@ -1,3 +1,4 @@ +v22.8.5.29-lts 2022-09-13 v22.8.4.7-lts 2022-08-31 v22.8.3.13-lts 2022-08-29 v22.8.2.11-lts 2022-08-23