diff --git a/.clang-tidy b/.clang-tidy index 4dd8b9859c9..7241c372319 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -23,9 +23,12 @@ Checks: '*, -bugprone-implicit-widening-of-multiplication-result, -bugprone-narrowing-conversions, -bugprone-not-null-terminated-result, + -bugprone-reserved-identifier, -bugprone-unchecked-optional-access, -cert-dcl16-c, + -cert-dcl37-c, + -cert-dcl51-cpp, -cert-err58-cpp, -cert-msc32-c, -cert-msc51-cpp, @@ -129,6 +132,7 @@ Checks: '*, -readability-function-cognitive-complexity, -readability-function-size, -readability-identifier-length, + -readability-identifier-naming, -readability-implicit-bool-conversion, -readability-isolate-declaration, -readability-magic-numbers, @@ -158,56 +162,28 @@ Checks: '*, WarningsAsErrors: '*' -# TODO: use dictionary syntax for CheckOptions when minimum clang-tidy level rose to 15 -# some-check.SomeOption: 'some value' -# instead of -# - key: some-check.SomeOption -# value: 'some value' CheckOptions: - - key: readability-identifier-naming.ClassCase - value: CamelCase - - key: readability-identifier-naming.EnumCase - value: CamelCase - - key: readability-identifier-naming.LocalVariableCase - value: lower_case - - key: readability-identifier-naming.StaticConstantCase - value: aNy_CasE - - key: readability-identifier-naming.MemberCase - value: lower_case - - key: readability-identifier-naming.PrivateMemberPrefix - value: '' - - key: readability-identifier-naming.ProtectedMemberPrefix - value: '' - - key: readability-identifier-naming.PublicMemberCase - value: lower_case - - key: readability-identifier-naming.MethodCase - value: camelBack - - key: readability-identifier-naming.PrivateMethodPrefix - value: '' - - key: readability-identifier-naming.ProtectedMethodPrefix - value: '' - - key: readability-identifier-naming.ParameterPackCase - value: lower_case - - key: readability-identifier-naming.StructCase - value: CamelCase - - key: readability-identifier-naming.TemplateTemplateParameterCase - value: CamelCase - - key: readability-identifier-naming.TemplateUsingCase - value: lower_case - - key: readability-identifier-naming.TypeTemplateParameterCase - value: CamelCase - - key: readability-identifier-naming.TypedefCase - value: CamelCase - - key: readability-identifier-naming.UnionCase - value: CamelCase - - key: readability-identifier-naming.UsingCase - value: CamelCase - - key: modernize-loop-convert.UseCxx20ReverseRanges - value: false - - key: performance-move-const-arg.CheckTriviallyCopyableMove - value: false - # Workaround clang-tidy bug: https://github.com/llvm/llvm-project/issues/46097 - - key: readability-identifier-naming.TypeTemplateParameterIgnoredRegexp - value: expr-type - - key: cppcoreguidelines-avoid-do-while.IgnoreMacros - value: true + readability-identifier-naming.ClassCase: CamelCase + readability-identifier-naming.EnumCase: CamelCase + readability-identifier-naming.LocalVariableCase: lower_case + readability-identifier-naming.StaticConstantCase: aNy_CasE + readability-identifier-naming.MemberCase: lower_case + readability-identifier-naming.PrivateMemberPrefix: '' + readability-identifier-naming.ProtectedMemberPrefix: '' + readability-identifier-naming.PublicMemberCase: lower_case + readability-identifier-naming.MethodCase: camelBack + readability-identifier-naming.PrivateMethodPrefix: '' + readability-identifier-naming.ProtectedMethodPrefix: '' + readability-identifier-naming.ParameterPackCase: lower_case + readability-identifier-naming.StructCase: CamelCase + readability-identifier-naming.TemplateTemplateParameterCase: CamelCase + readability-identifier-naming.TemplateUsingCase: lower_case + readability-identifier-naming.TypeTemplateParameterCase: CamelCase + readability-identifier-naming.TypedefCase: CamelCase + readability-identifier-naming.UnionCase: CamelCase + readability-identifier-naming.UsingCase: CamelCase + modernize-loop-convert.UseCxx20ReverseRanges: false + performance-move-const-arg.CheckTriviallyCopyableMove: false + # Workaround clang-tidy bug: https://github.com/llvm/llvm-project/issues/46097 + readability-identifier-naming.TypeTemplateParameterIgnoredRegexp: expr-type + cppcoreguidelines-avoid-do-while.IgnoreMacros: true diff --git a/.github/workflows/backport_branches.yml b/.github/workflows/backport_branches.yml index 110c06631c7..7cdf11fec0f 100644 --- a/.github/workflows/backport_branches.yml +++ b/.github/workflows/backport_branches.yml @@ -79,7 +79,7 @@ jobs: with: name: changed_images path: ${{ runner.temp }}/changed_images.json - CompatibilityCheck: + CompatibilityCheckX86: needs: [BuilderDebRelease] runs-on: [self-hosted, style-checker] steps: @@ -98,12 +98,43 @@ jobs: uses: actions/download-artifact@v3 with: path: ${{ env.REPORTS_PATH }} - - name: CompatibilityCheck + - name: CompatibilityCheckX86 run: | sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 compatibility_check.py + cd "$REPO_COPY/tests/ci" && python3 compatibility_check.py --check-name "Compatibility check (amd64)" --check-glibc --check-distributions + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" + CompatibilityCheckAarch64: + needs: [BuilderDebAarch64] + runs-on: [self-hosted, style-checker] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/compatibility_check + REPO_COPY=${{runner.temp}}/compatibility_check/ClickHouse + REPORTS_PATH=${{runner.temp}}/reports_dir + EOF + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: CompatibilityCheckAarch64 + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" && python3 compatibility_check.py --check-name "Compatibility check (aarch64)" --check-glibc - name: Cleanup if: always() run: | @@ -421,7 +452,8 @@ jobs: - name: Check docker clickhouse/clickhouse-server building run: | cd "$GITHUB_WORKSPACE/tests/ci" - python3 docker_server.py --release-type head --no-push + python3 docker_server.py --release-type head --no-push \ + --image-repo clickhouse/clickhouse-server --image-path docker/server python3 docker_server.py --release-type head --no-push --no-ubuntu \ --image-repo clickhouse/clickhouse-keeper --image-path docker/keeper - name: Cleanup @@ -741,7 +773,8 @@ jobs: - FunctionalStatefulTestDebug - StressTestTsan - IntegrationTestsRelease - - CompatibilityCheck + - CompatibilityCheckX86 + - CompatibilityCheckAarch64 runs-on: [self-hosted, style-checker] steps: - name: Check out repository code diff --git a/.github/workflows/cherry_pick.yml b/.github/workflows/cherry_pick.yml index 065e584182b..8d1e2055978 100644 --- a/.github/workflows/cherry_pick.yml +++ b/.github/workflows/cherry_pick.yml @@ -35,7 +35,6 @@ jobs: fetch-depth: 0 - name: Cherry pick run: | - sudo pip install GitPython cd "$GITHUB_WORKSPACE/tests/ci" python3 cherry_pick.py - name: Cleanup diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 7e045992dee..7c5e477ab60 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -110,7 +110,7 @@ jobs: docker ps --quiet | xargs --no-run-if-empty docker kill ||: docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" - CompatibilityCheck: + CompatibilityCheckX86: needs: [BuilderDebRelease] runs-on: [self-hosted, style-checker] steps: @@ -129,12 +129,43 @@ jobs: uses: actions/download-artifact@v3 with: path: ${{ env.REPORTS_PATH }} - - name: CompatibilityCheck + - name: CompatibilityCheckX86 run: | sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 compatibility_check.py + cd "$REPO_COPY/tests/ci" && python3 compatibility_check.py --check-name "Compatibility check (amd64)" --check-glibc --check-distributions + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" + CompatibilityCheckAarch64: + needs: [BuilderDebAarch64] + runs-on: [self-hosted, style-checker] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/compatibility_check + REPO_COPY=${{runner.temp}}/compatibility_check/ClickHouse + REPORTS_PATH=${{runner.temp}}/reports_dir + EOF + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: CompatibilityCheckAarch64 + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" && python3 compatibility_check.py --check-name "Compatibility check (aarch64)" --check-glibc - name: Cleanup if: always() run: | @@ -829,7 +860,8 @@ jobs: - name: Check docker clickhouse/clickhouse-server building run: | cd "$GITHUB_WORKSPACE/tests/ci" - python3 docker_server.py --release-type head + python3 docker_server.py --release-type head \ + --image-repo clickhouse/clickhouse-server --image-path docker/server python3 docker_server.py --release-type head --no-ubuntu \ --image-repo clickhouse/clickhouse-keeper --image-path docker/keeper - name: Cleanup @@ -3124,7 +3156,8 @@ jobs: - PerformanceComparisonX86-1 - PerformanceComparisonX86-2 - PerformanceComparisonX86-3 - - CompatibilityCheck + - CompatibilityCheckX86 + - CompatibilityCheckAarch64 - ASTFuzzerTestDebug - ASTFuzzerTestAsan - ASTFuzzerTestTsan diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 7d410f833c5..2f2c263df37 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -37,7 +37,6 @@ jobs: cd "$GITHUB_WORKSPACE/tests/ci" python3 run_check.py PythonUnitTests: - needs: CheckLabels runs-on: [self-hosted, style-checker] steps: - name: Check out repository code @@ -174,7 +173,7 @@ jobs: docker ps --quiet | xargs --no-run-if-empty docker kill ||: docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - CompatibilityCheck: + CompatibilityCheckX86: needs: [BuilderDebRelease] runs-on: [self-hosted, style-checker] steps: @@ -193,12 +192,43 @@ jobs: uses: actions/download-artifact@v3 with: path: ${{ env.REPORTS_PATH }} - - name: CompatibilityCheck + - name: CompatibilityCheckX86 run: | sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 compatibility_check.py + cd "$REPO_COPY/tests/ci" && python3 compatibility_check.py --check-name "Compatibility check (amd64)" --check-glibc --check-distributions + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" + CompatibilityCheckAarch64: + needs: [BuilderDebAarch64] + runs-on: [self-hosted, style-checker] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/compatibility_check + REPO_COPY=${{runner.temp}}/compatibility_check/ClickHouse + REPORTS_PATH=${{runner.temp}}/reports_dir + EOF + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: CompatibilityCheckAarch64 + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" && python3 compatibility_check.py --check-name "Compatibility check (aarch64)" --check-glibc - name: Cleanup if: always() run: | @@ -886,7 +916,8 @@ jobs: - name: Check docker clickhouse/clickhouse-server building run: | cd "$GITHUB_WORKSPACE/tests/ci" - python3 docker_server.py --release-type head --no-push + python3 docker_server.py --release-type head --no-push \ + --image-repo clickhouse/clickhouse-server --image-path docker/server python3 docker_server.py --release-type head --no-push --no-ubuntu \ --image-repo clickhouse/clickhouse-keeper --image-path docker/keeper - name: Cleanup @@ -4792,7 +4823,8 @@ jobs: - UnitTestsMsan - UnitTestsUBsan - UnitTestsReleaseClang - - CompatibilityCheck + - CompatibilityCheckX86 + - CompatibilityCheckAarch64 - IntegrationTestsFlakyCheck - SQLancerTestRelease - SQLancerTestDebug diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2ef05fe989b..73246af6dfc 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -7,15 +7,28 @@ on: # yamllint disable-line rule:truthy release: types: - published + workflow_dispatch: + inputs: + tag: + description: 'Release tag' + required: true + type: string jobs: ReleasePublish: runs-on: [self-hosted, style-checker] steps: + - name: Set tag from input + if: github.event_name == 'workflow_dispatch' + run: | + echo "GITHUB_TAG=${{ github.event.inputs.tag }}" >> "$GITHUB_ENV" + - name: Set tag from REF + if: github.event_name == 'release' + run: | + echo "GITHUB_TAG=${GITHUB_REF#refs/tags/}" >> "$GITHUB_ENV" - name: Deploy packages and assets run: | - GITHUB_TAG="${GITHUB_REF#refs/tags/}" - curl --silent --data '' \ + curl --silent --data '' --no-buffer \ '${{ secrets.PACKAGES_RELEASE_URL }}/release/'"${GITHUB_TAG}"'?binary=binary_darwin&binary=binary_darwin_aarch64&sync=true' ############################################################################################ ##################################### Docker images ####################################### @@ -23,16 +36,26 @@ jobs: DockerServerImages: runs-on: [self-hosted, style-checker] steps: + - name: Set tag from input + if: github.event_name == 'workflow_dispatch' + run: | + echo "GITHUB_TAG=${{ github.event.inputs.tag }}" >> "$GITHUB_ENV" + - name: Set tag from REF + if: github.event_name == 'release' + run: | + echo "GITHUB_TAG=${GITHUB_REF#refs/tags/}" >> "$GITHUB_ENV" - name: Check out repository code uses: ClickHouse/checkout@v1 with: clear-repository: true fetch-depth: 0 # otherwise we will have no version info + ref: ${{ env.GITHUB_TAG }} - name: Check docker clickhouse/clickhouse-server building run: | cd "$GITHUB_WORKSPACE/tests/ci" - python3 docker_server.py --release-type auto --version "${{ github.ref }}" - python3 docker_server.py --release-type auto --version "${{ github.ref }}" --no-ubuntu \ + python3 docker_server.py --release-type auto --version "$GITHUB_TAG" \ + --image-repo clickhouse/clickhouse-server --image-path docker/server + python3 docker_server.py --release-type auto --version "$GITHUB_TAG" --no-ubuntu \ --image-repo clickhouse/clickhouse-keeper --image-path docker/keeper - name: Cleanup if: always() diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 4d2a99c2106..e56a1fb58fc 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -71,7 +71,7 @@ jobs: with: name: changed_images path: ${{ runner.temp }}/changed_images.json - CompatibilityCheck: + CompatibilityCheckX86: needs: [BuilderDebRelease] runs-on: [self-hosted, style-checker] steps: @@ -90,12 +90,43 @@ jobs: uses: actions/download-artifact@v3 with: path: ${{ env.REPORTS_PATH }} - - name: CompatibilityCheck + - name: CompatibilityCheckX86 run: | sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 compatibility_check.py + cd "$REPO_COPY/tests/ci" && python3 compatibility_check.py --check-name "Compatibility check (amd64)" --check-glibc --check-distributions + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" + CompatibilityCheckAarch64: + needs: [BuilderDebAarch64] + runs-on: [self-hosted, style-checker] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/compatibility_check + REPO_COPY=${{runner.temp}}/compatibility_check/ClickHouse + REPORTS_PATH=${{runner.temp}}/reports_dir + EOF + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: CompatibilityCheckAarch64 + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" && python3 compatibility_check.py --check-name "Compatibility check (aarch64)" --check-glibc - name: Cleanup if: always() run: | @@ -494,7 +525,8 @@ jobs: - name: Check docker clickhouse/clickhouse-server building run: | cd "$GITHUB_WORKSPACE/tests/ci" - python3 docker_server.py --release-type head --no-push + python3 docker_server.py --release-type head --no-push \ + --image-repo clickhouse/clickhouse-server --image-path docker/server python3 docker_server.py --release-type head --no-push --no-ubuntu \ --image-repo clickhouse/clickhouse-keeper --image-path docker/keeper - name: Cleanup @@ -1947,7 +1979,8 @@ jobs: - IntegrationTestsTsan1 - IntegrationTestsTsan2 - IntegrationTestsTsan3 - - CompatibilityCheck + - CompatibilityCheckX86 + - CompatibilityCheckAarch64 runs-on: [self-hosted, style-checker] steps: - name: Check out repository code diff --git a/CMakeLists.txt b/CMakeLists.txt index cbb666b81c3..6accb2e09b7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -301,12 +301,12 @@ if (ENABLE_BUILD_PROFILING) endif () endif () -set (CMAKE_CXX_STANDARD 20) -set (CMAKE_CXX_EXTENSIONS ON) # Same as gnu++2a (ON) vs c++2a (OFF): https://cmake.org/cmake/help/latest/prop_tgt/CXX_EXTENSIONS.html +set (CMAKE_CXX_STANDARD 23) +set (CMAKE_CXX_EXTENSIONS OFF) set (CMAKE_CXX_STANDARD_REQUIRED ON) set (CMAKE_C_STANDARD 11) -set (CMAKE_C_EXTENSIONS ON) +set (CMAKE_C_EXTENSIONS ON) # required by most contribs written in C set (CMAKE_C_STANDARD_REQUIRED ON) if (COMPILER_GCC OR COMPILER_CLANG) diff --git a/README.md b/README.md index fcbe65e8223..17b4df154a9 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ curl https://clickhouse.com/ | sh * [Tutorial](https://clickhouse.com/docs/en/getting_started/tutorial/) shows how to set up and query a small ClickHouse cluster. * [Documentation](https://clickhouse.com/docs/en/) provides more in-depth information. * [YouTube channel](https://www.youtube.com/c/ClickHouseDB) has a lot of content about ClickHouse in video format. -* [Slack](https://join.slack.com/t/clickhousedb/shared_invite/zt-1gh9ds7f4-PgDhJAaF8ad5RbWBAAjzFg) and [Telegram](https://telegram.me/clickhouse_en) allow chatting with ClickHouse users in real-time. +* [Slack](https://clickhouse.com/slack) and [Telegram](https://telegram.me/clickhouse_en) allow chatting with ClickHouse users in real-time. * [Blog](https://clickhouse.com/blog/) contains various ClickHouse-related articles, as well as announcements and reports about events. * [Code Browser (Woboq)](https://clickhouse.com/codebrowser/ClickHouse/index.html) with syntax highlight and navigation. * [Code Browser (github.dev)](https://github.dev/ClickHouse/ClickHouse) with syntax highlight, powered by github.dev. diff --git a/base/base/CMakeLists.txt b/base/base/CMakeLists.txt index 64785d575c5..8ab3c8a0711 100644 --- a/base/base/CMakeLists.txt +++ b/base/base/CMakeLists.txt @@ -2,6 +2,10 @@ if (USE_CLANG_TIDY) set (CMAKE_CXX_CLANG_TIDY "${CLANG_TIDY_PATH}") endif () +# TODO: Remove this. We like to compile with C++23 (set by top-level CMakeLists) but Clang crashes with our libcxx +# when instantiated from JSON.cpp. Try again when libcxx(abi) and Clang are upgraded to 16. +set (CMAKE_CXX_STANDARD 20) + set (SRCS argsToConfig.cpp coverage.cpp diff --git a/base/base/TypeList.h b/base/base/TypeList.h index 244403b1c6b..310f0c0c586 100644 --- a/base/base/TypeList.h +++ b/base/base/TypeList.h @@ -4,7 +4,6 @@ #include #include #include "defines.h" -#include "TypePair.h" /// General-purpose typelist. Easy on compilation times as it does not use recursion. template @@ -28,7 +27,7 @@ namespace TypeListUtils /// In some contexts it's more handy to use functions in constexpr Root changeRoot(TypeList) { return {}; } template - constexpr void forEach(TypeList, F && f) { (std::forward(f)(Id{}), ...); } + constexpr void forEach(TypeList, F && f) { (std::forward(f)(TypeList{}), ...); } } template diff --git a/base/base/TypePair.h b/base/base/TypePair.h deleted file mode 100644 index 8c2f380618c..00000000000 --- a/base/base/TypePair.h +++ /dev/null @@ -1,4 +0,0 @@ -#pragma once - -template struct TypePair {}; -template struct Id {}; diff --git a/base/base/find_symbols.h b/base/base/find_symbols.h index 83f53773ae7..01efe7046bc 100644 --- a/base/base/find_symbols.h +++ b/base/base/find_symbols.h @@ -159,22 +159,22 @@ inline const char * find_first_symbols_sse42(const char * const begin, const cha #endif for (; pos < end; ++pos) - if ( (num_chars >= 1 && maybe_negate(*pos == c01)) - || (num_chars >= 2 && maybe_negate(*pos == c02)) - || (num_chars >= 3 && maybe_negate(*pos == c03)) - || (num_chars >= 4 && maybe_negate(*pos == c04)) - || (num_chars >= 5 && maybe_negate(*pos == c05)) - || (num_chars >= 6 && maybe_negate(*pos == c06)) - || (num_chars >= 7 && maybe_negate(*pos == c07)) - || (num_chars >= 8 && maybe_negate(*pos == c08)) - || (num_chars >= 9 && maybe_negate(*pos == c09)) - || (num_chars >= 10 && maybe_negate(*pos == c10)) - || (num_chars >= 11 && maybe_negate(*pos == c11)) - || (num_chars >= 12 && maybe_negate(*pos == c12)) - || (num_chars >= 13 && maybe_negate(*pos == c13)) - || (num_chars >= 14 && maybe_negate(*pos == c14)) - || (num_chars >= 15 && maybe_negate(*pos == c15)) - || (num_chars >= 16 && maybe_negate(*pos == c16))) + if ( (num_chars == 1 && maybe_negate(is_in(*pos))) + || (num_chars == 2 && maybe_negate(is_in(*pos))) + || (num_chars == 3 && maybe_negate(is_in(*pos))) + || (num_chars == 4 && maybe_negate(is_in(*pos))) + || (num_chars == 5 && maybe_negate(is_in(*pos))) + || (num_chars == 6 && maybe_negate(is_in(*pos))) + || (num_chars == 7 && maybe_negate(is_in(*pos))) + || (num_chars == 8 && maybe_negate(is_in(*pos))) + || (num_chars == 9 && maybe_negate(is_in(*pos))) + || (num_chars == 10 && maybe_negate(is_in(*pos))) + || (num_chars == 11 && maybe_negate(is_in(*pos))) + || (num_chars == 12 && maybe_negate(is_in(*pos))) + || (num_chars == 13 && maybe_negate(is_in(*pos))) + || (num_chars == 14 && maybe_negate(is_in(*pos))) + || (num_chars == 15 && maybe_negate(is_in(*pos))) + || (num_chars == 16 && maybe_negate(is_in(*pos)))) return pos; return return_mode == ReturnMode::End ? end : nullptr; } diff --git a/base/base/hex.h b/base/base/hex.h new file mode 100644 index 00000000000..e0c57f9dd42 --- /dev/null +++ b/base/base/hex.h @@ -0,0 +1,214 @@ +#pragma once + +#include +#include "types.h" + +/// Maps 0..15 to 0..9A..F or 0..9a..f correspondingly. + +constexpr inline std::string_view hex_digit_to_char_uppercase_table = "0123456789ABCDEF"; +constexpr inline std::string_view hex_digit_to_char_lowercase_table = "0123456789abcdef"; + +constexpr char hexDigitUppercase(unsigned char c) +{ + return hex_digit_to_char_uppercase_table[c]; +} +constexpr char hexDigitLowercase(unsigned char c) +{ + return hex_digit_to_char_lowercase_table[c]; +} + +/// Maps 0..255 to 00..FF or 00..ff correspondingly + +constexpr inline std::string_view hex_byte_to_char_uppercase_table = // + "000102030405060708090A0B0C0D0E0F" + "101112131415161718191A1B1C1D1E1F" + "202122232425262728292A2B2C2D2E2F" + "303132333435363738393A3B3C3D3E3F" + "404142434445464748494A4B4C4D4E4F" + "505152535455565758595A5B5C5D5E5F" + "606162636465666768696A6B6C6D6E6F" + "707172737475767778797A7B7C7D7E7F" + "808182838485868788898A8B8C8D8E8F" + "909192939495969798999A9B9C9D9E9F" + "A0A1A2A3A4A5A6A7A8A9AAABACADAEAF" + "B0B1B2B3B4B5B6B7B8B9BABBBCBDBEBF" + "C0C1C2C3C4C5C6C7C8C9CACBCCCDCECF" + "D0D1D2D3D4D5D6D7D8D9DADBDCDDDEDF" + "E0E1E2E3E4E5E6E7E8E9EAEBECEDEEEF" + "F0F1F2F3F4F5F6F7F8F9FAFBFCFDFEFF"; + +constexpr inline std::string_view hex_byte_to_char_lowercase_table = // + "000102030405060708090a0b0c0d0e0f" + "101112131415161718191a1b1c1d1e1f" + "202122232425262728292a2b2c2d2e2f" + "303132333435363738393a3b3c3d3e3f" + "404142434445464748494a4b4c4d4e4f" + "505152535455565758595a5b5c5d5e5f" + "606162636465666768696a6b6c6d6e6f" + "707172737475767778797a7b7c7d7e7f" + "808182838485868788898a8b8c8d8e8f" + "909192939495969798999a9b9c9d9e9f" + "a0a1a2a3a4a5a6a7a8a9aaabacadaeaf" + "b0b1b2b3b4b5b6b7b8b9babbbcbdbebf" + "c0c1c2c3c4c5c6c7c8c9cacbcccdcecf" + "d0d1d2d3d4d5d6d7d8d9dadbdcdddedf" + "e0e1e2e3e4e5e6e7e8e9eaebecedeeef" + "f0f1f2f3f4f5f6f7f8f9fafbfcfdfeff"; + +inline void writeHexByteUppercase(UInt8 byte, void * out) +{ + memcpy(out, &hex_byte_to_char_uppercase_table[static_cast(byte) * 2], 2); +} + +inline void writeHexByteLowercase(UInt8 byte, void * out) +{ + memcpy(out, &hex_byte_to_char_lowercase_table[static_cast(byte) * 2], 2); +} + +constexpr inline std::string_view bin_byte_to_char_table = // + "0000000000000001000000100000001100000100000001010000011000000111" + "0000100000001001000010100000101100001100000011010000111000001111" + "0001000000010001000100100001001100010100000101010001011000010111" + "0001100000011001000110100001101100011100000111010001111000011111" + "0010000000100001001000100010001100100100001001010010011000100111" + "0010100000101001001010100010101100101100001011010010111000101111" + "0011000000110001001100100011001100110100001101010011011000110111" + "0011100000111001001110100011101100111100001111010011111000111111" + "0100000001000001010000100100001101000100010001010100011001000111" + "0100100001001001010010100100101101001100010011010100111001001111" + "0101000001010001010100100101001101010100010101010101011001010111" + "0101100001011001010110100101101101011100010111010101111001011111" + "0110000001100001011000100110001101100100011001010110011001100111" + "0110100001101001011010100110101101101100011011010110111001101111" + "0111000001110001011100100111001101110100011101010111011001110111" + "0111100001111001011110100111101101111100011111010111111001111111" + "1000000010000001100000101000001110000100100001011000011010000111" + "1000100010001001100010101000101110001100100011011000111010001111" + "1001000010010001100100101001001110010100100101011001011010010111" + "1001100010011001100110101001101110011100100111011001111010011111" + "1010000010100001101000101010001110100100101001011010011010100111" + "1010100010101001101010101010101110101100101011011010111010101111" + "1011000010110001101100101011001110110100101101011011011010110111" + "1011100010111001101110101011101110111100101111011011111010111111" + "1100000011000001110000101100001111000100110001011100011011000111" + "1100100011001001110010101100101111001100110011011100111011001111" + "1101000011010001110100101101001111010100110101011101011011010111" + "1101100011011001110110101101101111011100110111011101111011011111" + "1110000011100001111000101110001111100100111001011110011011100111" + "1110100011101001111010101110101111101100111011011110111011101111" + "1111000011110001111100101111001111110100111101011111011011110111" + "1111100011111001111110101111101111111100111111011111111011111111"; + +inline void writeBinByte(UInt8 byte, void * out) +{ + memcpy(out, &bin_byte_to_char_table[static_cast(byte) * 8], 8); +} + +/// Produces hex representation of an unsigned int with leading zeros (for checksums) +template +inline void writeHexUIntImpl(TUInt uint_, char * out, std::string_view table) +{ + union + { + TUInt value; + UInt8 uint8[sizeof(TUInt)]; + }; + + value = uint_; + + for (size_t i = 0; i < sizeof(TUInt); ++i) + { + if constexpr (std::endian::native == std::endian::little) + memcpy(out + i * 2, &table[static_cast(uint8[sizeof(TUInt) - 1 - i]) * 2], 2); + else + memcpy(out + i * 2, &table[static_cast(uint8[i]) * 2], 2); + } +} + +template +inline void writeHexUIntUppercase(TUInt uint_, char * out) +{ + writeHexUIntImpl(uint_, out, hex_byte_to_char_uppercase_table); +} + +template +inline void writeHexUIntLowercase(TUInt uint_, char * out) +{ + writeHexUIntImpl(uint_, out, hex_byte_to_char_lowercase_table); +} + +template +std::string getHexUIntUppercase(TUInt uint_) +{ + std::string res(sizeof(TUInt) * 2, '\0'); + writeHexUIntUppercase(uint_, res.data()); + return res; +} + +template +std::string getHexUIntLowercase(TUInt uint_) +{ + std::string res(sizeof(TUInt) * 2, '\0'); + writeHexUIntLowercase(uint_, res.data()); + return res; +} + +/// Maps 0..9, A..F, a..f to 0..15. Other chars are mapped to implementation specific value. + +constexpr inline std::string_view hex_char_to_digit_table + = {"\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\xff\xff\xff\xff\xff\xff" //0-9 + "\xff\x0a\x0b\x0c\x0d\x0e\x0f\xff\xff\xff\xff\xff\xff\xff\xff\xff" //A-Z + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\x0a\x0b\x0c\x0d\x0e\x0f\xff\xff\xff\xff\xff\xff\xff\xff\xff" //a-z + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", + 256}; + +constexpr UInt8 unhex(char c) +{ + return hex_char_to_digit_table[static_cast(c)]; +} + +constexpr UInt8 unhex2(const char * data) +{ + return static_cast(unhex(data[0])) * 0x10 + static_cast(unhex(data[1])); +} + +constexpr UInt16 unhex4(const char * data) +{ + return static_cast(unhex(data[0])) * 0x1000 + static_cast(unhex(data[1])) * 0x100 + + static_cast(unhex(data[2])) * 0x10 + static_cast(unhex(data[3])); +} + +template +constexpr TUInt unhexUInt(const char * data) +{ + TUInt res = 0; + if constexpr ((sizeof(TUInt) <= 8) || ((sizeof(TUInt) % 8) != 0)) + { + for (size_t i = 0; i < sizeof(TUInt) * 2; ++i, ++data) + { + res <<= 4; + res += unhex(*data); + } + } + else + { + for (size_t i = 0; i < sizeof(TUInt) / 8; ++i, data += 16) + { + res <<= 64; + res += unhexUInt(data); + } + } + return res; +} diff --git a/base/base/interpolate.h b/base/base/interpolate.h new file mode 100644 index 00000000000..1d4fc0b6257 --- /dev/null +++ b/base/base/interpolate.h @@ -0,0 +1,13 @@ +#pragma once +#include +#include + +/** Linear interpolation in logarithmic coordinates. + * Exponential interpolation is related to linear interpolation + * exactly in same way as geometric mean is related to arithmetic mean. + */ +constexpr double interpolateExponential(double min, double max, double ratio) +{ + assert(min > 0 && ratio >= 0 && ratio <= 1); + return min * std::pow(max / min, ratio); +} diff --git a/base/poco/Data/ODBC/include/Poco/Data/ODBC/Extractor.h b/base/poco/Data/ODBC/include/Poco/Data/ODBC/Extractor.h index 82e2f895638..3914f33df76 100644 --- a/base/poco/Data/ODBC/include/Poco/Data/ODBC/Extractor.h +++ b/base/poco/Data/ODBC/include/Poco/Data/ODBC/Extractor.h @@ -466,7 +466,7 @@ namespace Data bool extractManualImpl(std::size_t pos, T & val, SQLSMALLINT cType) { SQLRETURN rc = 0; - T value = (T)0; + T value; resizeLengths(pos); diff --git a/base/poco/Foundation/include/Poco/Message.h b/base/poco/Foundation/include/Poco/Message.h index f78b3ebb6a1..e8f04888ab4 100644 --- a/base/poco/Foundation/include/Poco/Message.h +++ b/base/poco/Foundation/include/Poco/Message.h @@ -105,6 +105,8 @@ public: const std::string & getText() const; /// Returns the text of the message. + void appendText(const std::string & text); + void setPriority(Priority prio); /// Sets the priority of the message. diff --git a/base/poco/Foundation/src/Message.cpp b/base/poco/Foundation/src/Message.cpp index 0dfe4323134..663c96e47a2 100644 --- a/base/poco/Foundation/src/Message.cpp +++ b/base/poco/Foundation/src/Message.cpp @@ -27,8 +27,7 @@ Message::Message(): _tid(0), _file(0), _line(0), - _pMap(0), - _fmt_str(0) + _pMap(0) { init(); } @@ -157,6 +156,12 @@ void Message::setText(const std::string& text) } +void Message::appendText(const std::string & text) +{ + _text.append(text); +} + + void Message::setPriority(Priority prio) { _prio = prio; diff --git a/base/poco/MongoDB/include/Poco/MongoDB/Connection.h b/base/poco/MongoDB/include/Poco/MongoDB/Connection.h index de669aa90dd..dcb813b75bc 100644 --- a/base/poco/MongoDB/include/Poco/MongoDB/Connection.h +++ b/base/poco/MongoDB/include/Poco/MongoDB/Connection.h @@ -90,6 +90,9 @@ namespace MongoDB Poco::Net::SocketAddress address() const; /// Returns the address of the MongoDB server. + + const std::string & uri() const; + /// Returns the uri on which the connection was made. void connect(const std::string & hostAndPort); /// Connects to the given MongoDB server. @@ -148,6 +151,7 @@ namespace MongoDB private: Poco::Net::SocketAddress _address; Poco::Net::StreamSocket _socket; + std::string _uri; }; @@ -158,6 +162,10 @@ namespace MongoDB { return _address; } + inline const std::string & Connection::uri() const + { + return _uri; + } } diff --git a/base/poco/MongoDB/src/Connection.cpp b/base/poco/MongoDB/src/Connection.cpp index 56bb192cec2..38c31d2250a 100644 --- a/base/poco/MongoDB/src/Connection.cpp +++ b/base/poco/MongoDB/src/Connection.cpp @@ -145,68 +145,155 @@ void Connection::connect(const Poco::Net::StreamSocket& socket) void Connection::connect(const std::string& uri, SocketFactory& socketFactory) { - Poco::URI theURI(uri); - if (theURI.getScheme() != "mongodb") throw Poco::UnknownURISchemeException(uri); + std::vector strAddresses; + std::string newURI; - std::string userInfo = theURI.getUserInfo(); - std::string host = theURI.getHost(); - Poco::UInt16 port = theURI.getPort(); - if (port == 0) port = 27017; + if (uri.find(',') != std::string::npos) + { + size_t pos; + size_t head = 0; + if ((pos = uri.find("@")) != std::string::npos) + { + head = pos + 1; + } + else if ((pos = uri.find("://")) != std::string::npos) + { + head = pos + 3; + } - std::string databaseName = theURI.getPath(); - if (!databaseName.empty() && databaseName[0] == '/') databaseName.erase(0, 1); - if (databaseName.empty()) databaseName = "admin"; + std::string tempstr; + std::string::const_iterator it = uri.begin(); + it += head; + size_t tail = head; + for (;it != uri.end() && *it != '?' && *it != '/'; ++it) + { + tempstr += *it; + tail++; + } - bool ssl = false; - Poco::Timespan connectTimeout; - Poco::Timespan socketTimeout; - std::string authMechanism = Database::AUTH_SCRAM_SHA1; + it = tempstr.begin(); + std::string token; + for (;it != tempstr.end(); ++it) + { + if (*it == ',') + { + newURI = uri.substr(0, head) + token + uri.substr(tail, uri.length()); + strAddresses.push_back(newURI); + token = ""; + } + else + { + token += *it; + } + } + newURI = uri.substr(0, head) + token + uri.substr(tail, uri.length()); + strAddresses.push_back(newURI); + } + else + { + strAddresses.push_back(uri); + } - Poco::URI::QueryParameters params = theURI.getQueryParameters(); - for (Poco::URI::QueryParameters::const_iterator it = params.begin(); it != params.end(); ++it) - { - if (it->first == "ssl") - { - ssl = (it->second == "true"); - } - else if (it->first == "connectTimeoutMS") - { - connectTimeout = static_cast(1000)*Poco::NumberParser::parse(it->second); - } - else if (it->first == "socketTimeoutMS") - { - socketTimeout = static_cast(1000)*Poco::NumberParser::parse(it->second); - } - else if (it->first == "authMechanism") - { - authMechanism = it->second; - } - } + newURI = strAddresses.front(); + Poco::URI theURI(newURI); + if (theURI.getScheme() != "mongodb") throw Poco::UnknownURISchemeException(uri); - connect(socketFactory.createSocket(host, port, connectTimeout, ssl)); + std::string userInfo = theURI.getUserInfo(); + std::string databaseName = theURI.getPath(); + if (!databaseName.empty() && databaseName[0] == '/') databaseName.erase(0, 1); + if (databaseName.empty()) databaseName = "admin"; - if (socketTimeout > 0) - { - _socket.setSendTimeout(socketTimeout); - _socket.setReceiveTimeout(socketTimeout); - } + bool ssl = false; + Poco::Timespan connectTimeout; + Poco::Timespan socketTimeout; + std::string authMechanism = Database::AUTH_SCRAM_SHA1; + std::string readPreference="primary"; - if (!userInfo.empty()) - { - std::string username; - std::string password; - std::string::size_type pos = userInfo.find(':'); - if (pos != std::string::npos) - { - username.assign(userInfo, 0, pos++); - password.assign(userInfo, pos, userInfo.size() - pos); - } - else username = userInfo; + Poco::URI::QueryParameters params = theURI.getQueryParameters(); + for (Poco::URI::QueryParameters::const_iterator it = params.begin(); it != params.end(); ++it) + { + if (it->first == "ssl") + { + ssl = (it->second == "true"); + } + else if (it->first == "connectTimeoutMS") + { + connectTimeout = static_cast(1000)*Poco::NumberParser::parse(it->second); + } + else if (it->first == "socketTimeoutMS") + { + socketTimeout = static_cast(1000)*Poco::NumberParser::parse(it->second); + } + else if (it->first == "authMechanism") + { + authMechanism = it->second; + } + else if (it->first == "readPreference") + { + readPreference= it->second; + } + } - Database database(databaseName); - if (!database.authenticate(*this, username, password, authMechanism)) - throw Poco::NoPermissionException(Poco::format("Access to MongoDB database %s denied for user %s", databaseName, username)); - } + for (std::vector::const_iterator it = strAddresses.cbegin();it != strAddresses.cend(); ++it) + { + newURI = *it; + theURI = Poco::URI(newURI); + + std::string host = theURI.getHost(); + Poco::UInt16 port = theURI.getPort(); + if (port == 0) port = 27017; + + connect(socketFactory.createSocket(host, port, connectTimeout, ssl)); + _uri = newURI; + if (socketTimeout > 0) + { + _socket.setSendTimeout(socketTimeout); + _socket.setReceiveTimeout(socketTimeout); + } + if (strAddresses.size() > 1) + { + Poco::MongoDB::QueryRequest request("admin.$cmd"); + request.setNumberToReturn(1); + request.selector().add("isMaster", 1); + Poco::MongoDB::ResponseMessage response; + + sendRequest(request, response); + _uri = newURI; + if (!response.documents().empty()) + { + Poco::MongoDB::Document::Ptr doc = response.documents()[0]; + if (doc->get("ismaster") && readPreference == "primary") + { + break; + } + else if (!doc->get("ismaster") && readPreference == "secondary") + { + break; + } + else if (it + 1 == strAddresses.cend()) + { + throw Poco::URISyntaxException(uri); + } + } + } + } + if (!userInfo.empty()) + { + std::string username; + std::string password; + std::string::size_type pos = userInfo.find(':'); + if (pos != std::string::npos) + { + username.assign(userInfo, 0, pos++); + password.assign(userInfo, pos, userInfo.size() - pos); + } + else username = userInfo; + + Database database(databaseName); + + if (!database.authenticate(*this, username, password, authMechanism)) + throw Poco::NoPermissionException(Poco::format("Access to MongoDB database %s denied for user %s", databaseName, username)); + } } diff --git a/cmake/tools.cmake b/cmake/tools.cmake index 4d4d741cc3a..4e1954f27f7 100644 --- a/cmake/tools.cmake +++ b/cmake/tools.cmake @@ -50,15 +50,18 @@ endif () string (REGEX MATCHALL "[0-9]+" COMPILER_VERSION_LIST ${CMAKE_CXX_COMPILER_VERSION}) list (GET COMPILER_VERSION_LIST 0 COMPILER_VERSION_MAJOR) -# Example values: `lld-10`, `gold`. +# Example values: `lld-10` option (LINKER_NAME "Linker name or full path") +if (LINKER_NAME MATCHES "gold") + message (FATAL_ERROR "Linking with gold is unsupported. Please use lld.") +endif () + # s390x doesnt support lld if (NOT ARCH_S390X) if (NOT LINKER_NAME) if (COMPILER_GCC) find_program (LLD_PATH NAMES "ld.lld") - find_program (GOLD_PATH NAMES "ld.gold") elseif (COMPILER_CLANG) # llvm lld is a generic driver. # Invoke ld.lld (Unix), ld64.lld (macOS), lld-link (Windows), wasm-ld (WebAssembly) instead @@ -67,13 +70,11 @@ if (NOT ARCH_S390X) elseif (OS_DARWIN) find_program (LLD_PATH NAMES "ld64.lld-${COMPILER_VERSION_MAJOR}" "ld64.lld") endif () - find_program (GOLD_PATH NAMES "ld.gold" "gold") endif () endif() endif() if ((OS_LINUX OR OS_DARWIN) AND NOT LINKER_NAME) - # prefer lld linker over gold or ld on linux and macos if (LLD_PATH) if (COMPILER_GCC) # GCC driver requires one of supported linker names like "lld". @@ -83,17 +84,6 @@ if ((OS_LINUX OR OS_DARWIN) AND NOT LINKER_NAME) set (LINKER_NAME ${LLD_PATH}) endif () endif () - - if (NOT LINKER_NAME) - if (GOLD_PATH) - message (FATAL_ERROR "Linking with gold is unsupported. Please use lld.") - if (COMPILER_GCC) - set (LINKER_NAME "gold") - else () - set (LINKER_NAME ${GOLD_PATH}) - endif () - endif () - endif () endif () # TODO: allow different linker on != OS_LINUX diff --git a/contrib/arrow-cmake/CMakeLists.txt b/contrib/arrow-cmake/CMakeLists.txt index ae6f270a768..4181f916d63 100644 --- a/contrib/arrow-cmake/CMakeLists.txt +++ b/contrib/arrow-cmake/CMakeLists.txt @@ -115,6 +115,13 @@ configure_file("${ORC_SOURCE_SRC_DIR}/Adaptor.hh.in" "${ORC_BUILD_INCLUDE_DIR}/A # ARROW_ORC + adapters/orc/CMakefiles set(ORC_SRCS + "${CMAKE_CURRENT_BINARY_DIR}/orc_proto.pb.h" + "${ORC_SOURCE_SRC_DIR}/sargs/ExpressionTree.cc" + "${ORC_SOURCE_SRC_DIR}/sargs/Literal.cc" + "${ORC_SOURCE_SRC_DIR}/sargs/PredicateLeaf.cc" + "${ORC_SOURCE_SRC_DIR}/sargs/SargsApplier.cc" + "${ORC_SOURCE_SRC_DIR}/sargs/SearchArgument.cc" + "${ORC_SOURCE_SRC_DIR}/sargs/TruthValue.cc" "${ORC_SOURCE_SRC_DIR}/Exceptions.cc" "${ORC_SOURCE_SRC_DIR}/OrcFile.cc" "${ORC_SOURCE_SRC_DIR}/Reader.cc" @@ -129,13 +136,20 @@ set(ORC_SRCS "${ORC_SOURCE_SRC_DIR}/MemoryPool.cc" "${ORC_SOURCE_SRC_DIR}/RLE.cc" "${ORC_SOURCE_SRC_DIR}/RLEv1.cc" - "${ORC_SOURCE_SRC_DIR}/RLEv2.cc" + "${ORC_SOURCE_SRC_DIR}/RleDecoderV2.cc" + "${ORC_SOURCE_SRC_DIR}/RleEncoderV2.cc" + "${ORC_SOURCE_SRC_DIR}/RLEV2Util.cc" "${ORC_SOURCE_SRC_DIR}/Statistics.cc" "${ORC_SOURCE_SRC_DIR}/StripeStream.cc" "${ORC_SOURCE_SRC_DIR}/Timezone.cc" "${ORC_SOURCE_SRC_DIR}/TypeImpl.cc" "${ORC_SOURCE_SRC_DIR}/Vector.cc" "${ORC_SOURCE_SRC_DIR}/Writer.cc" + "${ORC_SOURCE_SRC_DIR}/Adaptor.cc" + "${ORC_SOURCE_SRC_DIR}/BloomFilter.cc" + "${ORC_SOURCE_SRC_DIR}/Murmur3.cc" + "${ORC_SOURCE_SRC_DIR}/BlockBuffer.cc" + "${ORC_SOURCE_SRC_DIR}/wrap/orc-proto-wrapper.cc" "${ORC_SOURCE_SRC_DIR}/io/InputStream.cc" "${ORC_SOURCE_SRC_DIR}/io/OutputStream.cc" "${ORC_ADDITION_SOURCE_DIR}/orc_proto.pb.cc" @@ -358,6 +372,9 @@ SET(ARROW_SRCS "${LIBRARY_DIR}/util/compression_zlib.cc" ${ARROW_SRCS}) add_definitions(-DARROW_WITH_ZSTD) SET(ARROW_SRCS "${LIBRARY_DIR}/util/compression_zstd.cc" ${ARROW_SRCS}) +add_definitions(-DARROW_WITH_BROTLI) +SET(ARROW_SRCS "${LIBRARY_DIR}/util/compression_brotli.cc" ${ARROW_SRCS}) + add_library(_arrow ${ARROW_SRCS}) @@ -372,6 +389,7 @@ target_link_libraries(_arrow PRIVATE ch_contrib::snappy ch_contrib::zlib ch_contrib::zstd + ch_contrib::brotli ) target_link_libraries(_arrow PUBLIC _orc) diff --git a/contrib/grpc-cmake/CMakeLists.txt b/contrib/grpc-cmake/CMakeLists.txt index b1ed7e464b6..b4cf0ad5e66 100644 --- a/contrib/grpc-cmake/CMakeLists.txt +++ b/contrib/grpc-cmake/CMakeLists.txt @@ -48,6 +48,9 @@ set(gRPC_ABSL_PROVIDER "clickhouse" CACHE STRING "" FORCE) # We don't want to build C# extensions. set(gRPC_BUILD_CSHARP_EXT OFF) +# TODO: Remove this. We generally like to compile with C++23 but grpc isn't ready yet. +set (CMAKE_CXX_STANDARD 20) + set(_gRPC_CARES_LIBRARIES ch_contrib::c-ares) set(gRPC_CARES_PROVIDER "clickhouse" CACHE STRING "" FORCE) add_subdirectory("${_gRPC_SOURCE_DIR}" "${_gRPC_BINARY_DIR}") diff --git a/contrib/krb5 b/contrib/krb5 index f8262a1b548..9453aec0d50 160000 --- a/contrib/krb5 +++ b/contrib/krb5 @@ -1 +1 @@ -Subproject commit f8262a1b548eb29d97e059260042036255d07f8d +Subproject commit 9453aec0d50e5aff9b189051611b321b40935d02 diff --git a/contrib/krb5-cmake/CMakeLists.txt b/contrib/krb5-cmake/CMakeLists.txt index ceaa270ad85..93b90c15201 100644 --- a/contrib/krb5-cmake/CMakeLists.txt +++ b/contrib/krb5-cmake/CMakeLists.txt @@ -160,6 +160,8 @@ set(ALL_SRCS # "${KRB5_SOURCE_DIR}/lib/gssapi/spnego/negoex_trace.c" + "${KRB5_SOURCE_DIR}/lib/crypto/builtin/kdf.c" + "${KRB5_SOURCE_DIR}/lib/crypto/builtin/cmac.c" "${KRB5_SOURCE_DIR}/lib/crypto/krb/prng.c" "${KRB5_SOURCE_DIR}/lib/crypto/krb/enc_dk_cmac.c" # "${KRB5_SOURCE_DIR}/lib/crypto/krb/crc32.c" @@ -183,7 +185,6 @@ set(ALL_SRCS "${KRB5_SOURCE_DIR}/lib/crypto/krb/block_size.c" "${KRB5_SOURCE_DIR}/lib/crypto/krb/string_to_key.c" "${KRB5_SOURCE_DIR}/lib/crypto/krb/verify_checksum.c" - "${KRB5_SOURCE_DIR}/lib/crypto/krb/crypto_libinit.c" "${KRB5_SOURCE_DIR}/lib/crypto/krb/derive.c" "${KRB5_SOURCE_DIR}/lib/crypto/krb/random_to_key.c" "${KRB5_SOURCE_DIR}/lib/crypto/krb/verify_checksum_iov.c" @@ -217,9 +218,7 @@ set(ALL_SRCS "${KRB5_SOURCE_DIR}/lib/crypto/krb/s2k_rc4.c" "${KRB5_SOURCE_DIR}/lib/crypto/krb/valid_cksumtype.c" "${KRB5_SOURCE_DIR}/lib/crypto/krb/nfold.c" - "${KRB5_SOURCE_DIR}/lib/crypto/krb/prng_fortuna.c" "${KRB5_SOURCE_DIR}/lib/crypto/krb/encrypt_length.c" - "${KRB5_SOURCE_DIR}/lib/crypto/krb/cmac.c" "${KRB5_SOURCE_DIR}/lib/crypto/krb/keyblocks.c" "${KRB5_SOURCE_DIR}/lib/crypto/krb/prf_rc4.c" "${KRB5_SOURCE_DIR}/lib/crypto/krb/s2k_pbkdf2.c" @@ -228,11 +227,11 @@ set(ALL_SRCS "${KRB5_SOURCE_DIR}/lib/crypto/openssl/enc_provider/rc4.c" "${KRB5_SOURCE_DIR}/lib/crypto/openssl/enc_provider/des3.c" #"${KRB5_SOURCE_DIR}/lib/crypto/openssl/enc_provider/camellia.c" + "${KRB5_SOURCE_DIR}/lib/crypto/openssl/cmac.c" "${KRB5_SOURCE_DIR}/lib/crypto/openssl/sha256.c" "${KRB5_SOURCE_DIR}/lib/crypto/openssl/hmac.c" + "${KRB5_SOURCE_DIR}/lib/crypto/openssl/kdf.c" "${KRB5_SOURCE_DIR}/lib/crypto/openssl/pbkdf2.c" - "${KRB5_SOURCE_DIR}/lib/crypto/openssl/init.c" - "${KRB5_SOURCE_DIR}/lib/crypto/openssl/stubs.c" # "${KRB5_SOURCE_DIR}/lib/crypto/openssl/hash_provider/hash_crc32.c" "${KRB5_SOURCE_DIR}/lib/crypto/openssl/hash_provider/hash_evp.c" "${KRB5_SOURCE_DIR}/lib/crypto/openssl/des/des_keys.c" @@ -312,7 +311,6 @@ set(ALL_SRCS "${KRB5_SOURCE_DIR}/lib/krb5/krb/allow_weak.c" "${KRB5_SOURCE_DIR}/lib/krb5/krb/mk_rep.c" "${KRB5_SOURCE_DIR}/lib/krb5/krb/mk_priv.c" - "${KRB5_SOURCE_DIR}/lib/krb5/krb/s4u_authdata.c" "${KRB5_SOURCE_DIR}/lib/krb5/krb/preauth_otp.c" "${KRB5_SOURCE_DIR}/lib/krb5/krb/init_keyblock.c" "${KRB5_SOURCE_DIR}/lib/krb5/krb/ser_addr.c" @@ -688,6 +686,7 @@ target_include_directories(_krb5 PRIVATE target_compile_definitions(_krb5 PRIVATE KRB5_PRIVATE + CRYPTO_OPENSSL _GSS_STATIC_LINK=1 KRB5_DEPRECATED=1 LOCALEDIR="/usr/local/share/locale" diff --git a/contrib/libfarmhash/CMakeLists.txt b/contrib/libfarmhash/CMakeLists.txt index a0533a93f17..436bc3d0108 100644 --- a/contrib/libfarmhash/CMakeLists.txt +++ b/contrib/libfarmhash/CMakeLists.txt @@ -6,6 +6,10 @@ if (MSVC) target_compile_definitions (_farmhash PRIVATE FARMHASH_NO_BUILTIN_EXPECT=1) endif () +if (ARCH_S390X) + add_compile_definitions(WORDS_BIGENDIAN) +endif () + target_include_directories (_farmhash BEFORE PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) add_library(ch_contrib::farmhash ALIAS _farmhash) diff --git a/contrib/orc b/contrib/orc index f9a393ed243..c5d7755ba0b 160000 --- a/contrib/orc +++ b/contrib/orc @@ -1 +1 @@ -Subproject commit f9a393ed2433a60034795284f82d093b348f2102 +Subproject commit c5d7755ba0b9a95631c8daea4d094101f26ec761 diff --git a/docker/keeper/Dockerfile b/docker/keeper/Dockerfile index 09395befdad..34c1406b687 100644 --- a/docker/keeper/Dockerfile +++ b/docker/keeper/Dockerfile @@ -29,7 +29,7 @@ RUN arch=${TARGETARCH:-amd64} \ esac ARG REPOSITORY="https://s3.amazonaws.com/clickhouse-builds/22.4/31c367d3cd3aefd316778601ff6565119fe36682/package_release" -ARG VERSION="23.2.1.2537" +ARG VERSION="23.2.4.12" ARG PACKAGES="clickhouse-keeper" # user/group precreated explicitly with fixed uid/gid on purpose. diff --git a/docker/server/Dockerfile.alpine b/docker/server/Dockerfile.alpine index 472f25eed2d..f4ca498a7e7 100644 --- a/docker/server/Dockerfile.alpine +++ b/docker/server/Dockerfile.alpine @@ -33,7 +33,7 @@ RUN arch=${TARGETARCH:-amd64} \ # lts / testing / prestable / etc ARG REPO_CHANNEL="stable" ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}" -ARG VERSION="23.2.1.2537" +ARG VERSION="23.2.4.12" ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static" # user/group precreated explicitly with fixed uid/gid on purpose. diff --git a/docker/server/Dockerfile.ubuntu b/docker/server/Dockerfile.ubuntu index 5dbb244c298..13b3ebdb01c 100644 --- a/docker/server/Dockerfile.ubuntu +++ b/docker/server/Dockerfile.ubuntu @@ -22,7 +22,7 @@ RUN sed -i "s|http://archive.ubuntu.com|${apt_archive}|g" /etc/apt/sources.list ARG REPO_CHANNEL="stable" ARG REPOSITORY="deb https://packages.clickhouse.com/deb ${REPO_CHANNEL} main" -ARG VERSION="23.2.1.2537" +ARG VERSION="23.2.4.12" ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static" # set non-empty deb_location_url url to create a docker image diff --git a/docker/test/stress/run.sh b/docker/test/stress/run.sh index 15f58d6c3a3..314e9c2acfd 100644 --- a/docker/test/stress/run.sh +++ b/docker/test/stress/run.sh @@ -44,6 +44,8 @@ if [ "$is_tsan_build" -eq "0" ]; then fi export ZOOKEEPER_FAULT_INJECTION=1 +# Initial run without S3 to create system.*_log on local file system to make it +# available for dump via clickhouse-local configure azurite-blob --blobHost 0.0.0.0 --blobPort 10000 --debug /azurite_log & diff --git a/docker/test/upgrade/run.sh b/docker/test/upgrade/run.sh index df32e2833e7..b9abe5b51fe 100644 --- a/docker/test/upgrade/run.sh +++ b/docker/test/upgrade/run.sh @@ -49,19 +49,26 @@ echo -e "Successfully cloned previous release tests$OK" >> /test_output/test_res echo -e "Successfully downloaded previous release packages$OK" >> /test_output/test_results.tsv # Make upgrade check more funny by forcing Ordinary engine for system database -mkdir /var/lib/clickhouse/metadata +mkdir -p /var/lib/clickhouse/metadata echo "ATTACH DATABASE system ENGINE=Ordinary" > /var/lib/clickhouse/metadata/system.sql # Install previous release packages install_packages previous_release_package_folder -# Start server from previous release -# Let's enable S3 storage by default -export USE_S3_STORAGE_FOR_MERGE_TREE=1 -# Previous version may not be ready for fault injections -export ZOOKEEPER_FAULT_INJECTION=0 +# Initial run without S3 to create system.*_log on local file system to make it +# available for dump via clickhouse-local configure +start +stop +mv /var/log/clickhouse-server/clickhouse-server.log /var/log/clickhouse-server/clickhouse-server.initial.log + +# force_sync=false doesn't work correctly on some older versions +sudo cat /etc/clickhouse-server/config.d/keeper_port.xml \ + | sed "s|false|true|" \ + > /etc/clickhouse-server/config.d/keeper_port.xml.tmp +sudo mv /etc/clickhouse-server/config.d/keeper_port.xml.tmp /etc/clickhouse-server/config.d/keeper_port.xml + # But we still need default disk because some tables loaded only into it sudo cat /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml \ | sed "s|
s3
|
s3
default|" \ @@ -69,6 +76,13 @@ sudo cat /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml \ sudo chown clickhouse /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml sudo chgrp clickhouse /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml +# Start server from previous release +# Let's enable S3 storage by default +export USE_S3_STORAGE_FOR_MERGE_TREE=1 +# Previous version may not be ready for fault injections +export ZOOKEEPER_FAULT_INJECTION=0 +configure + start clickhouse-client --query="SELECT 'Server version: ', version()" @@ -161,8 +175,10 @@ rg -Fav -e "Code: 236. DB::Exception: Cancelled merging parts" \ -e "Authentication failed" \ -e "Cannot flush" \ -e "Container already exists" \ - /var/log/clickhouse-server/clickhouse-server.upgrade.log | zgrep -Fa "" > /test_output/upgrade_error_messages.txt \ - && echo -e "Error message in clickhouse-server.log (see upgrade_error_messages.txt)$FAIL$(head_escaped /test_output/bc_check_error_messages.txt)" \ + clickhouse-server.upgrade.log \ + | grep -av -e "_repl_01111_.*Mapping for table with UUID" \ + | zgrep -Fa "" > /test_output/upgrade_error_messages.txt \ + && echo -e "Error message in clickhouse-server.log (see upgrade_error_messages.txt)$FAIL$(head_escaped /test_output/upgrade_error_messages.txt)" \ >> /test_output/test_results.tsv \ || echo -e "No Error messages after server upgrade$OK" >> /test_output/test_results.tsv @@ -176,8 +192,6 @@ tar -chf /test_output/coordination.tar /var/lib/clickhouse/coordination ||: collect_query_and_trace_logs -check_oom_in_dmesg - mv /var/log/clickhouse-server/stderr.log /test_output/ # Write check result into check_status.tsv diff --git a/docs/changelogs/v22.12.4.76-stable.md b/docs/changelogs/v22.12.4.76-stable.md new file mode 100644 index 00000000000..79569ff841e --- /dev/null +++ b/docs/changelogs/v22.12.4.76-stable.md @@ -0,0 +1,55 @@ +--- +sidebar_position: 1 +sidebar_label: 2023 +--- + +# 2023 Changelog + +### ClickHouse release v22.12.4.76-stable (cb5772db805) FIXME as compared to v22.12.3.5-stable (893de538f02) + +#### Performance Improvement +* Backported in [#45704](https://github.com/ClickHouse/ClickHouse/issues/45704): Fixed performance of short `SELECT` queries that read from tables with large number of`Array`/`Map`/`Nested` columns. [#45630](https://github.com/ClickHouse/ClickHouse/pull/45630) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#46378](https://github.com/ClickHouse/ClickHouse/issues/46378): Fix too big memory usage for vertical merges on non-remote disk. Respect `max_insert_delayed_streams_for_parallel_write` for the remote disk. [#46275](https://github.com/ClickHouse/ClickHouse/pull/46275) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). + +#### Bug Fix +* Backported in [#45672](https://github.com/ClickHouse/ClickHouse/issues/45672): Fix wiping sensitive info in logs. [#45603](https://github.com/ClickHouse/ClickHouse/pull/45603) ([Vitaly Baranov](https://github.com/vitlibar)). + +#### Build/Testing/Packaging Improvement +* Backported in [#45200](https://github.com/ClickHouse/ClickHouse/issues/45200): Fix zookeeper downloading, update the version, and optimize the image size. [#44853](https://github.com/ClickHouse/ClickHouse/pull/44853) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Backported in [#46116](https://github.com/ClickHouse/ClickHouse/issues/46116): Remove the dependency on the `adduser` tool from the packages, because we don't use it. This fixes [#44934](https://github.com/ClickHouse/ClickHouse/issues/44934). [#45011](https://github.com/ClickHouse/ClickHouse/pull/45011) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#46035](https://github.com/ClickHouse/ClickHouse/issues/46035): Add systemd.service file for clickhouse-keeper. Fixes [#44293](https://github.com/ClickHouse/ClickHouse/issues/44293). [#45568](https://github.com/ClickHouse/ClickHouse/pull/45568) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Backported in [#46484](https://github.com/ClickHouse/ClickHouse/issues/46484): Get rid of unnecessary build for standalone clickhouse-keeper. [#46367](https://github.com/ClickHouse/ClickHouse/pull/46367) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Backported in [#46509](https://github.com/ClickHouse/ClickHouse/issues/46509): Some time ago the ccache compression was changed to `zst`, but `gz` archives are downloaded by default. It fixes it by prioritizing zst archive. [#46490](https://github.com/ClickHouse/ClickHouse/pull/46490) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Backported in [#47058](https://github.com/ClickHouse/ClickHouse/issues/47058): Fix error during server startup on old distros (e.g. Amazon Linux 2) and on ARM that glibc 2.28 symbols are not found. [#47008](https://github.com/ClickHouse/ClickHouse/pull/47008) ([Robert Schulze](https://github.com/rschu1ze)). + +#### Bug Fix (user-visible misbehavior in official stable or prestable release) + +* Backported in [#45904](https://github.com/ClickHouse/ClickHouse/issues/45904): Fixed bug with non-parsable default value for EPHEMERAL column in table metadata. [#44026](https://github.com/ClickHouse/ClickHouse/pull/44026) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). +* Backported in [#45321](https://github.com/ClickHouse/ClickHouse/issues/45321): Fixed a bug in normalization of a `DEFAULT` expression in `CREATE TABLE` statement. The second argument of function `in` (or the right argument of operator `IN`) might be replaced with the result of its evaluation during CREATE query execution. Fixes [#44496](https://github.com/ClickHouse/ClickHouse/issues/44496). [#44547](https://github.com/ClickHouse/ClickHouse/pull/44547) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#45000](https://github.com/ClickHouse/ClickHouse/issues/45000): Another fix for `Cannot read all data` error which could happen while reading `LowCardinality` dictionary from remote fs. Fixes [#44709](https://github.com/ClickHouse/ClickHouse/issues/44709). [#44875](https://github.com/ClickHouse/ClickHouse/pull/44875) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#45553](https://github.com/ClickHouse/ClickHouse/issues/45553): Fix `SELECT ... FROM system.dictionaries` exception when there is a dictionary with a bad structure (e.g. incorrect type in xml config). [#45399](https://github.com/ClickHouse/ClickHouse/pull/45399) ([Aleksei Filatov](https://github.com/aalexfvk)). +* Backported in [#46226](https://github.com/ClickHouse/ClickHouse/issues/46226): A couple of seg faults have been reported around `c-ares`. All of the recent stack traces observed fail on inserting into `std::unodered_set<>`. I believe I have found the root cause of this, it seems to be unprocessed queries. Prior to this PR, CH calls `poll` to wait on the file descriptors in the `c-ares` channel. According to the [poll docs](https://man7.org/linux/man-pages/man2/poll.2.html), a negative return value means an error has ocurred. Because of this, we would abort the execution and return failure. The problem is that `poll` will also return a negative value if a system interrupt occurs. A system interrupt does not mean the processing has failed or ended, but we would abort it anyways because we were checking for negative values. Once the execution is aborted, the whole stack is destroyed, which includes the `std::unordered_set` passed to the `void *` parameter of the c-ares callback. Once c-ares completed the request, the callback would be invoked and would access an invalid memory address causing a segfault. [#45629](https://github.com/ClickHouse/ClickHouse/pull/45629) ([Arthur Passos](https://github.com/arthurpassos)). +* Backported in [#46218](https://github.com/ClickHouse/ClickHouse/issues/46218): Fix reading of non existing nested columns with multiple level in compact parts. [#46045](https://github.com/ClickHouse/ClickHouse/pull/46045) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#46446](https://github.com/ClickHouse/ClickHouse/issues/46446): Fix possible `LOGICAL_ERROR` in asynchronous inserts with invalid data sent in format `VALUES`. [#46350](https://github.com/ClickHouse/ClickHouse/pull/46350) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#46678](https://github.com/ClickHouse/ClickHouse/issues/46678): Fix an invalid processing of constant `LowCardinality` argument in function `arrayMap`. This bug could lead to a segfault in release, and logical error `Bad cast` in debug build. [#46569](https://github.com/ClickHouse/ClickHouse/pull/46569) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#46872](https://github.com/ClickHouse/ClickHouse/issues/46872): Fix a bug in the `Map` data type. This closes [#46855](https://github.com/ClickHouse/ClickHouse/issues/46855). [#46856](https://github.com/ClickHouse/ClickHouse/pull/46856) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#46954](https://github.com/ClickHouse/ClickHouse/issues/46954): Fix result of LIKE predicates which translate to substring searches and contain quoted non-LIKE metacharacters. [#46875](https://github.com/ClickHouse/ClickHouse/pull/46875) ([Robert Schulze](https://github.com/rschu1ze)). + +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Automatically merge green backport PRs and green approved PRs [#41110](https://github.com/ClickHouse/ClickHouse/pull/41110) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Improve release scripts [#45074](https://github.com/ClickHouse/ClickHouse/pull/45074) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix wrong approved_at, simplify conditions [#45302](https://github.com/ClickHouse/ClickHouse/pull/45302) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Get rid of artifactory in favor of r2 + ch-repos-manager [#45421](https://github.com/ClickHouse/ClickHouse/pull/45421) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Another attempt to fix automerge, or at least to have debug footprint [#45476](https://github.com/ClickHouse/ClickHouse/pull/45476) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Trim refs/tags/ from GITHUB_TAG in release workflow [#45636](https://github.com/ClickHouse/ClickHouse/pull/45636) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Add check for running workflows to merge_pr.py [#45803](https://github.com/ClickHouse/ClickHouse/pull/45803) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Get rid of progress timestamps in release publishing [#45818](https://github.com/ClickHouse/ClickHouse/pull/45818) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Add necessary dependency for sanitizers [#45959](https://github.com/ClickHouse/ClickHouse/pull/45959) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Add helping logging to auto-merge script [#46080](https://github.com/ClickHouse/ClickHouse/pull/46080) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix write buffer destruction order for vertical merge. [#46205](https://github.com/ClickHouse/ClickHouse/pull/46205) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Improve install_check.py [#46458](https://github.com/ClickHouse/ClickHouse/pull/46458) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix dependencies for InstallPackagesTestAarch64 [#46597](https://github.com/ClickHouse/ClickHouse/pull/46597) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Get rid of legacy DocsReleaseChecks [#46665](https://github.com/ClickHouse/ClickHouse/pull/46665) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Reduce updates of Mergeable Check [#46781](https://github.com/ClickHouse/ClickHouse/pull/46781) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). + diff --git a/docs/changelogs/v22.12.5.34-stable.md b/docs/changelogs/v22.12.5.34-stable.md new file mode 100644 index 00000000000..95befaa88ff --- /dev/null +++ b/docs/changelogs/v22.12.5.34-stable.md @@ -0,0 +1,29 @@ +--- +sidebar_position: 1 +sidebar_label: 2023 +--- + +# 2023 Changelog + +### ClickHouse release v22.12.5.34-stable (b82d6401ca1) FIXME as compared to v22.12.4.76-stable (cb5772db805) + +#### Improvement +* Backported in [#46983](https://github.com/ClickHouse/ClickHouse/issues/46983): - Apply `ALTER TABLE table_name ON CLUSTER cluster MOVE PARTITION|PART partition_expr TO DISK|VOLUME 'disk_name'` to all replicas. Because `ALTER TABLE t MOVE` is not replicated. [#46402](https://github.com/ClickHouse/ClickHouse/pull/46402) ([lizhuoyu5](https://github.com/lzydmxy)). + +#### Bug Fix (user-visible misbehavior in official stable or prestable release) + +* Backported in [#45729](https://github.com/ClickHouse/ClickHouse/issues/45729): Fix key description when encountering duplicate primary keys. This can happen in projections. See [#45590](https://github.com/ClickHouse/ClickHouse/issues/45590) for details. [#45686](https://github.com/ClickHouse/ClickHouse/pull/45686) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#46398](https://github.com/ClickHouse/ClickHouse/issues/46398): Fix `SYSTEM UNFREEZE` queries failing with the exception `CANNOT_PARSE_INPUT_ASSERTION_FAILED`. [#46325](https://github.com/ClickHouse/ClickHouse/pull/46325) ([Aleksei Filatov](https://github.com/aalexfvk)). +* Backported in [#46903](https://github.com/ClickHouse/ClickHouse/issues/46903): - Fix incorrect alias recursion in QueryNormalizer. [#46609](https://github.com/ClickHouse/ClickHouse/pull/46609) ([Raúl Marín](https://github.com/Algunenano)). +* Backported in [#47210](https://github.com/ClickHouse/ClickHouse/issues/47210): `INSERT` queries through native TCP protocol and HTTP protocol were not canceled correctly in some cases. It could lead to a partially applied query if a client canceled the query, or if a client died or, in rare cases, on network errors. As a result, it could lead to not working deduplication. Fixes [#27667](https://github.com/ClickHouse/ClickHouse/issues/27667) and [#45377](https://github.com/ClickHouse/ClickHouse/issues/45377). [#46681](https://github.com/ClickHouse/ClickHouse/pull/46681) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#47157](https://github.com/ClickHouse/ClickHouse/issues/47157): - Fix arithmetic operations in aggregate optimization with `min` and `max`. [#46705](https://github.com/ClickHouse/ClickHouse/pull/46705) ([Duc Canh Le](https://github.com/canhld94)). +* Backported in [#46881](https://github.com/ClickHouse/ClickHouse/issues/46881): Fix MSan report in the `maxIntersections` function. This closes [#43126](https://github.com/ClickHouse/ClickHouse/issues/43126). [#46847](https://github.com/ClickHouse/ClickHouse/pull/46847) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#47359](https://github.com/ClickHouse/ClickHouse/issues/47359): Fix possible deadlock on distributed query cancellation. [#47161](https://github.com/ClickHouse/ClickHouse/pull/47161) ([Kruglov Pavel](https://github.com/Avogar)). + +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Use /etc/default/clickhouse in systemd too [#47003](https://github.com/ClickHouse/ClickHouse/pull/47003) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Update typing for a new PyGithub version [#47123](https://github.com/ClickHouse/ClickHouse/pull/47123) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Follow-up to [#46681](https://github.com/ClickHouse/ClickHouse/issues/46681) [#47284](https://github.com/ClickHouse/ClickHouse/pull/47284) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Add a manual trigger for release workflow [#47302](https://github.com/ClickHouse/ClickHouse/pull/47302) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). + diff --git a/docs/changelogs/v22.8.14.53-lts.md b/docs/changelogs/v22.8.14.53-lts.md new file mode 100644 index 00000000000..5978080fa3a --- /dev/null +++ b/docs/changelogs/v22.8.14.53-lts.md @@ -0,0 +1,40 @@ +--- +sidebar_position: 1 +sidebar_label: 2023 +--- + +# 2023 Changelog + +### ClickHouse release v22.8.14.53-lts (4ea67c40077) FIXME as compared to v22.8.13.20-lts (e4817946d18) + +#### Performance Improvement +* Backported in [#45845](https://github.com/ClickHouse/ClickHouse/issues/45845): Fixed performance of short `SELECT` queries that read from tables with large number of`Array`/`Map`/`Nested` columns. [#45630](https://github.com/ClickHouse/ClickHouse/pull/45630) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#46374](https://github.com/ClickHouse/ClickHouse/issues/46374): Fix too big memory usage for vertical merges on non-remote disk. Respect `max_insert_delayed_streams_for_parallel_write` for the remote disk. [#46275](https://github.com/ClickHouse/ClickHouse/pull/46275) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#46358](https://github.com/ClickHouse/ClickHouse/issues/46358): Allow using Vertical merge algorithm with parts in Compact format. This will allow ClickHouse server to use much less memory for background operations. This closes [#46084](https://github.com/ClickHouse/ClickHouse/issues/46084). [#46282](https://github.com/ClickHouse/ClickHouse/pull/46282) ([Anton Popov](https://github.com/CurtizJ)). + +#### Build/Testing/Packaging Improvement +* Backported in [#46112](https://github.com/ClickHouse/ClickHouse/issues/46112): Remove the dependency on the `adduser` tool from the packages, because we don't use it. This fixes [#44934](https://github.com/ClickHouse/ClickHouse/issues/44934). [#45011](https://github.com/ClickHouse/ClickHouse/pull/45011) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#46482](https://github.com/ClickHouse/ClickHouse/issues/46482): Get rid of unnecessary build for standalone clickhouse-keeper. [#46367](https://github.com/ClickHouse/ClickHouse/pull/46367) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Backported in [#46505](https://github.com/ClickHouse/ClickHouse/issues/46505): Some time ago the ccache compression was changed to `zst`, but `gz` archives are downloaded by default. It fixes it by prioritizing zst archive. [#46490](https://github.com/ClickHouse/ClickHouse/pull/46490) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). + +#### Bug Fix (user-visible misbehavior in official stable or prestable release) + +* Backported in [#45908](https://github.com/ClickHouse/ClickHouse/issues/45908): Fixed bug with non-parsable default value for EPHEMERAL column in table metadata. [#44026](https://github.com/ClickHouse/ClickHouse/pull/44026) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). +* Backported in [#46238](https://github.com/ClickHouse/ClickHouse/issues/46238): A couple of seg faults have been reported around `c-ares`. All of the recent stack traces observed fail on inserting into `std::unodered_set<>`. I believe I have found the root cause of this, it seems to be unprocessed queries. Prior to this PR, CH calls `poll` to wait on the file descriptors in the `c-ares` channel. According to the [poll docs](https://man7.org/linux/man-pages/man2/poll.2.html), a negative return value means an error has ocurred. Because of this, we would abort the execution and return failure. The problem is that `poll` will also return a negative value if a system interrupt occurs. A system interrupt does not mean the processing has failed or ended, but we would abort it anyways because we were checking for negative values. Once the execution is aborted, the whole stack is destroyed, which includes the `std::unordered_set` passed to the `void *` parameter of the c-ares callback. Once c-ares completed the request, the callback would be invoked and would access an invalid memory address causing a segfault. [#45629](https://github.com/ClickHouse/ClickHouse/pull/45629) ([Arthur Passos](https://github.com/arthurpassos)). +* Backported in [#45727](https://github.com/ClickHouse/ClickHouse/issues/45727): Fix key description when encountering duplicate primary keys. This can happen in projections. See [#45590](https://github.com/ClickHouse/ClickHouse/issues/45590) for details. [#45686](https://github.com/ClickHouse/ClickHouse/pull/45686) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#46394](https://github.com/ClickHouse/ClickHouse/issues/46394): Fix `SYSTEM UNFREEZE` queries failing with the exception `CANNOT_PARSE_INPUT_ASSERTION_FAILED`. [#46325](https://github.com/ClickHouse/ClickHouse/pull/46325) ([Aleksei Filatov](https://github.com/aalexfvk)). +* Backported in [#46442](https://github.com/ClickHouse/ClickHouse/issues/46442): Fix possible `LOGICAL_ERROR` in asynchronous inserts with invalid data sent in format `VALUES`. [#46350](https://github.com/ClickHouse/ClickHouse/pull/46350) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#46674](https://github.com/ClickHouse/ClickHouse/issues/46674): Fix an invalid processing of constant `LowCardinality` argument in function `arrayMap`. This bug could lead to a segfault in release, and logical error `Bad cast` in debug build. [#46569](https://github.com/ClickHouse/ClickHouse/pull/46569) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#46879](https://github.com/ClickHouse/ClickHouse/issues/46879): Fix MSan report in the `maxIntersections` function. This closes [#43126](https://github.com/ClickHouse/ClickHouse/issues/43126). [#46847](https://github.com/ClickHouse/ClickHouse/pull/46847) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#46871](https://github.com/ClickHouse/ClickHouse/issues/46871): Fix a bug in the `Map` data type. This closes [#46855](https://github.com/ClickHouse/ClickHouse/issues/46855). [#46856](https://github.com/ClickHouse/ClickHouse/pull/46856) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Another attempt to fix automerge, or at least to have debug footprint [#45476](https://github.com/ClickHouse/ClickHouse/pull/45476) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Add check for running workflows to merge_pr.py [#45803](https://github.com/ClickHouse/ClickHouse/pull/45803) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Get rid of progress timestamps in release publishing [#45818](https://github.com/ClickHouse/ClickHouse/pull/45818) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Add necessary dependency for sanitizers [#45959](https://github.com/ClickHouse/ClickHouse/pull/45959) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Add helping logging to auto-merge script [#46080](https://github.com/ClickHouse/ClickHouse/pull/46080) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix write buffer destruction order for vertical merge. [#46205](https://github.com/ClickHouse/ClickHouse/pull/46205) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Get rid of legacy DocsReleaseChecks [#46665](https://github.com/ClickHouse/ClickHouse/pull/46665) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). + diff --git a/docs/changelogs/v22.8.15.23-lts.md b/docs/changelogs/v22.8.15.23-lts.md new file mode 100644 index 00000000000..096a504c9c2 --- /dev/null +++ b/docs/changelogs/v22.8.15.23-lts.md @@ -0,0 +1,28 @@ +--- +sidebar_position: 1 +sidebar_label: 2023 +--- + +# 2023 Changelog + +### ClickHouse release v22.8.15.23-lts (d36fa168bbf) FIXME as compared to v22.8.14.53-lts (4ea67c40077) + +#### Improvement +* Backported in [#46981](https://github.com/ClickHouse/ClickHouse/issues/46981): - Apply `ALTER TABLE table_name ON CLUSTER cluster MOVE PARTITION|PART partition_expr TO DISK|VOLUME 'disk_name'` to all replicas. Because `ALTER TABLE t MOVE` is not replicated. [#46402](https://github.com/ClickHouse/ClickHouse/pull/46402) ([lizhuoyu5](https://github.com/lzydmxy)). + +#### Bug Fix +* Backported in [#47336](https://github.com/ClickHouse/ClickHouse/issues/47336): Sometimes after changing a role that could be not reflected on the access rights of a user who uses that role. This PR fixes that. [#46772](https://github.com/ClickHouse/ClickHouse/pull/46772) ([Vitaly Baranov](https://github.com/vitlibar)). + +#### Bug Fix (user-visible misbehavior in official stable or prestable release) + +* Backported in [#46901](https://github.com/ClickHouse/ClickHouse/issues/46901): - Fix incorrect alias recursion in QueryNormalizer. [#46609](https://github.com/ClickHouse/ClickHouse/pull/46609) ([Raúl Marín](https://github.com/Algunenano)). +* Backported in [#47156](https://github.com/ClickHouse/ClickHouse/issues/47156): - Fix arithmetic operations in aggregate optimization with `min` and `max`. [#46705](https://github.com/ClickHouse/ClickHouse/pull/46705) ([Duc Canh Le](https://github.com/canhld94)). +* Backported in [#46987](https://github.com/ClickHouse/ClickHouse/issues/46987): Fix result of LIKE predicates which translate to substring searches and contain quoted non-LIKE metacharacters. [#46875](https://github.com/ClickHouse/ClickHouse/pull/46875) ([Robert Schulze](https://github.com/rschu1ze)). +* Backported in [#47357](https://github.com/ClickHouse/ClickHouse/issues/47357): Fix possible deadlock on distributed query cancellation. [#47161](https://github.com/ClickHouse/ClickHouse/pull/47161) ([Kruglov Pavel](https://github.com/Avogar)). + +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Reduce updates of Mergeable Check [#46781](https://github.com/ClickHouse/ClickHouse/pull/46781) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Update typing for a new PyGithub version [#47123](https://github.com/ClickHouse/ClickHouse/pull/47123) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Add a manual trigger for release workflow [#47302](https://github.com/ClickHouse/ClickHouse/pull/47302) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). + diff --git a/docs/changelogs/v23.1.4.58-stable.md b/docs/changelogs/v23.1.4.58-stable.md new file mode 100644 index 00000000000..d1ffe87f58e --- /dev/null +++ b/docs/changelogs/v23.1.4.58-stable.md @@ -0,0 +1,47 @@ +--- +sidebar_position: 1 +sidebar_label: 2023 +--- + +# 2023 Changelog + +### ClickHouse release v23.1.4.58-stable (9ed562163a5) FIXME as compared to v23.1.3.5-stable (548b494bcce) + +#### Performance Improvement +* Backported in [#46380](https://github.com/ClickHouse/ClickHouse/issues/46380): Fix too big memory usage for vertical merges on non-remote disk. Respect `max_insert_delayed_streams_for_parallel_write` for the remote disk. [#46275](https://github.com/ClickHouse/ClickHouse/pull/46275) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). + +#### Improvement +* Backported in [#46985](https://github.com/ClickHouse/ClickHouse/issues/46985): - Apply `ALTER TABLE table_name ON CLUSTER cluster MOVE PARTITION|PART partition_expr TO DISK|VOLUME 'disk_name'` to all replicas. Because `ALTER TABLE t MOVE` is not replicated. [#46402](https://github.com/ClickHouse/ClickHouse/pull/46402) ([lizhuoyu5](https://github.com/lzydmxy)). +* Backported in [#46778](https://github.com/ClickHouse/ClickHouse/issues/46778): Backward compatibility for T64 codec support for IPv4. [#46747](https://github.com/ClickHouse/ClickHouse/pull/46747) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). +* Backported in [#47020](https://github.com/ClickHouse/ClickHouse/issues/47020): Allow IPv4 in range(). [#46995](https://github.com/ClickHouse/ClickHouse/pull/46995) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). + +#### Build/Testing/Packaging Improvement +* Backported in [#46031](https://github.com/ClickHouse/ClickHouse/issues/46031): Add systemd.service file for clickhouse-keeper. Fixes [#44293](https://github.com/ClickHouse/ClickHouse/issues/44293). [#45568](https://github.com/ClickHouse/ClickHouse/pull/45568) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Backported in [#46477](https://github.com/ClickHouse/ClickHouse/issues/46477): Get rid of unnecessary build for standalone clickhouse-keeper. [#46367](https://github.com/ClickHouse/ClickHouse/pull/46367) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Backported in [#46511](https://github.com/ClickHouse/ClickHouse/issues/46511): Some time ago the ccache compression was changed to `zst`, but `gz` archives are downloaded by default. It fixes it by prioritizing zst archive. [#46490](https://github.com/ClickHouse/ClickHouse/pull/46490) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). + +#### Bug Fix (user-visible misbehavior in official stable or prestable release) + +* Backported in [#46228](https://github.com/ClickHouse/ClickHouse/issues/46228): A couple of seg faults have been reported around `c-ares`. All of the recent stack traces observed fail on inserting into `std::unodered_set<>`. I believe I have found the root cause of this, it seems to be unprocessed queries. Prior to this PR, CH calls `poll` to wait on the file descriptors in the `c-ares` channel. According to the [poll docs](https://man7.org/linux/man-pages/man2/poll.2.html), a negative return value means an error has ocurred. Because of this, we would abort the execution and return failure. The problem is that `poll` will also return a negative value if a system interrupt occurs. A system interrupt does not mean the processing has failed or ended, but we would abort it anyways because we were checking for negative values. Once the execution is aborted, the whole stack is destroyed, which includes the `std::unordered_set` passed to the `void *` parameter of the c-ares callback. Once c-ares completed the request, the callback would be invoked and would access an invalid memory address causing a segfault. [#45629](https://github.com/ClickHouse/ClickHouse/pull/45629) ([Arthur Passos](https://github.com/arthurpassos)). +* Backported in [#46967](https://github.com/ClickHouse/ClickHouse/issues/46967): Backward compatibility - allow implicit narrowing conversion from UInt64 to IPv4 - required for "INSERT ... VALUES ..." expression. [#45865](https://github.com/ClickHouse/ClickHouse/pull/45865) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). +* Backported in [#46220](https://github.com/ClickHouse/ClickHouse/issues/46220): Fix reading of non existing nested columns with multiple level in compact parts. [#46045](https://github.com/ClickHouse/ClickHouse/pull/46045) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#46751](https://github.com/ClickHouse/ClickHouse/issues/46751): Follow-up fix for Replace domain IP types (IPv4, IPv6) with native https://github.com/ClickHouse/ClickHouse/pull/43221. [#46087](https://github.com/ClickHouse/ClickHouse/pull/46087) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). +* Backported in [#46448](https://github.com/ClickHouse/ClickHouse/issues/46448): Fix possible `LOGICAL_ERROR` in asynchronous inserts with invalid data sent in format `VALUES`. [#46350](https://github.com/ClickHouse/ClickHouse/pull/46350) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#46680](https://github.com/ClickHouse/ClickHouse/issues/46680): Fix an invalid processing of constant `LowCardinality` argument in function `arrayMap`. This bug could lead to a segfault in release, and logical error `Bad cast` in debug build. [#46569](https://github.com/ClickHouse/ClickHouse/pull/46569) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#46873](https://github.com/ClickHouse/ClickHouse/issues/46873): Fix a bug in the `Map` data type. This closes [#46855](https://github.com/ClickHouse/ClickHouse/issues/46855). [#46856](https://github.com/ClickHouse/ClickHouse/pull/46856) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#46956](https://github.com/ClickHouse/ClickHouse/issues/46956): Fix result of LIKE predicates which translate to substring searches and contain quoted non-LIKE metacharacters. [#46875](https://github.com/ClickHouse/ClickHouse/pull/46875) ([Robert Schulze](https://github.com/rschu1ze)). + +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Another attempt to fix automerge, or at least to have debug footprint [#45476](https://github.com/ClickHouse/ClickHouse/pull/45476) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Support DELETE ON CLUSTER [#45786](https://github.com/ClickHouse/ClickHouse/pull/45786) ([Alexander Gololobov](https://github.com/davenger)). +* Add check for running workflows to merge_pr.py [#45803](https://github.com/ClickHouse/ClickHouse/pull/45803) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Add necessary dependency for sanitizers [#45959](https://github.com/ClickHouse/ClickHouse/pull/45959) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Add helping logging to auto-merge script [#46080](https://github.com/ClickHouse/ClickHouse/pull/46080) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix write buffer destruction order for vertical merge. [#46205](https://github.com/ClickHouse/ClickHouse/pull/46205) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Wait for background tasks in ~UploadHelper [#46334](https://github.com/ClickHouse/ClickHouse/pull/46334) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Improve install_check.py [#46458](https://github.com/ClickHouse/ClickHouse/pull/46458) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix dependencies for InstallPackagesTestAarch64 [#46597](https://github.com/ClickHouse/ClickHouse/pull/46597) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Get rid of legacy DocsReleaseChecks [#46665](https://github.com/ClickHouse/ClickHouse/pull/46665) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Reduce updates of Mergeable Check [#46781](https://github.com/ClickHouse/ClickHouse/pull/46781) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). + diff --git a/docs/changelogs/v23.1.5.24-stable.md b/docs/changelogs/v23.1.5.24-stable.md new file mode 100644 index 00000000000..1c2c127a8c3 --- /dev/null +++ b/docs/changelogs/v23.1.5.24-stable.md @@ -0,0 +1,28 @@ +--- +sidebar_position: 1 +sidebar_label: 2023 +--- + +# 2023 Changelog + +### ClickHouse release v23.1.5.24-stable (0e51b53ba99) FIXME as compared to v23.1.4.58-stable (9ed562163a5) + +#### Build/Testing/Packaging Improvement +* Backported in [#47060](https://github.com/ClickHouse/ClickHouse/issues/47060): Fix error during server startup on old distros (e.g. Amazon Linux 2) and on ARM that glibc 2.28 symbols are not found. [#47008](https://github.com/ClickHouse/ClickHouse/pull/47008) ([Robert Schulze](https://github.com/rschu1ze)). + +#### Bug Fix (user-visible misbehavior in official stable or prestable release) + +* Backported in [#46401](https://github.com/ClickHouse/ClickHouse/issues/46401): Fix `SYSTEM UNFREEZE` queries failing with the exception `CANNOT_PARSE_INPUT_ASSERTION_FAILED`. [#46325](https://github.com/ClickHouse/ClickHouse/pull/46325) ([Aleksei Filatov](https://github.com/aalexfvk)). +* Backported in [#46905](https://github.com/ClickHouse/ClickHouse/issues/46905): - Fix incorrect alias recursion in QueryNormalizer. [#46609](https://github.com/ClickHouse/ClickHouse/pull/46609) ([Raúl Marín](https://github.com/Algunenano)). +* Backported in [#47211](https://github.com/ClickHouse/ClickHouse/issues/47211): `INSERT` queries through native TCP protocol and HTTP protocol were not canceled correctly in some cases. It could lead to a partially applied query if a client canceled the query, or if a client died or, in rare cases, on network errors. As a result, it could lead to not working deduplication. Fixes [#27667](https://github.com/ClickHouse/ClickHouse/issues/27667) and [#45377](https://github.com/ClickHouse/ClickHouse/issues/45377). [#46681](https://github.com/ClickHouse/ClickHouse/pull/46681) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#47118](https://github.com/ClickHouse/ClickHouse/issues/47118): - Fix arithmetic operations in aggregate optimization with `min` and `max`. [#46705](https://github.com/ClickHouse/ClickHouse/pull/46705) ([Duc Canh Le](https://github.com/canhld94)). +* Backported in [#46883](https://github.com/ClickHouse/ClickHouse/issues/46883): Fix MSan report in the `maxIntersections` function. This closes [#43126](https://github.com/ClickHouse/ClickHouse/issues/43126). [#46847](https://github.com/ClickHouse/ClickHouse/pull/46847) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#47361](https://github.com/ClickHouse/ClickHouse/issues/47361): Fix possible deadlock on distributed query cancellation. [#47161](https://github.com/ClickHouse/ClickHouse/pull/47161) ([Kruglov Pavel](https://github.com/Avogar)). + +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Use /etc/default/clickhouse in systemd too [#47003](https://github.com/ClickHouse/ClickHouse/pull/47003) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Update typing for a new PyGithub version [#47123](https://github.com/ClickHouse/ClickHouse/pull/47123) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Follow-up to [#46681](https://github.com/ClickHouse/ClickHouse/issues/46681) [#47284](https://github.com/ClickHouse/ClickHouse/pull/47284) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Add a manual trigger for release workflow [#47302](https://github.com/ClickHouse/ClickHouse/pull/47302) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). + diff --git a/docs/changelogs/v23.2.2.20-stable.md b/docs/changelogs/v23.2.2.20-stable.md new file mode 100644 index 00000000000..60aeaa66cbf --- /dev/null +++ b/docs/changelogs/v23.2.2.20-stable.md @@ -0,0 +1,30 @@ +--- +sidebar_position: 1 +sidebar_label: 2023 +--- + +# 2023 Changelog + +### ClickHouse release v23.2.2.20-stable (f6c269c8df2) FIXME as compared to v23.2.1.2537-stable (52bf836e03a) + +#### Improvement +* Backported in [#46914](https://github.com/ClickHouse/ClickHouse/issues/46914): Allow PREWHERE for Merge with different DEFAULT expression for column. [#46831](https://github.com/ClickHouse/ClickHouse/pull/46831) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#47022](https://github.com/ClickHouse/ClickHouse/issues/47022): Allow IPv4 in range(). [#46995](https://github.com/ClickHouse/ClickHouse/pull/46995) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). + +#### Bug Fix +* Backported in [#46828](https://github.com/ClickHouse/ClickHouse/issues/46828): Combined PREWHERE column accumulated from multiple PREWHERE in some cases didn't contain 0's from previous steps. The fix is to apply final filter if we know that it wasn't applied from more than 1 last step. [#46785](https://github.com/ClickHouse/ClickHouse/pull/46785) ([Alexander Gololobov](https://github.com/davenger)). + +#### Build/Testing/Packaging Improvement +* Backported in [#47062](https://github.com/ClickHouse/ClickHouse/issues/47062): Fix error during server startup on old distros (e.g. Amazon Linux 2) and on ARM that glibc 2.28 symbols are not found. [#47008](https://github.com/ClickHouse/ClickHouse/pull/47008) ([Robert Schulze](https://github.com/rschu1ze)). + +#### Bug Fix (user-visible misbehavior in official stable or prestable release) + +* Backported in [#46895](https://github.com/ClickHouse/ClickHouse/issues/46895): Fixed a bug in automatic retries of `DROP TABLE` query with `ReplicatedMergeTree` tables and `Atomic` databases. In rare cases it could lead to `Can't get data for node /zk_path/log_pointer` and `The specified key does not exist` errors if ZooKeeper session expired during DROP and a new replicated table with the same path in ZooKeeper was created in parallel. [#46384](https://github.com/ClickHouse/ClickHouse/pull/46384) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#46865](https://github.com/ClickHouse/ClickHouse/issues/46865): Fix a bug in the `Map` data type. This closes [#46855](https://github.com/ClickHouse/ClickHouse/issues/46855). [#46856](https://github.com/ClickHouse/ClickHouse/pull/46856) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#46958](https://github.com/ClickHouse/ClickHouse/issues/46958): Fix result of LIKE predicates which translate to substring searches and contain quoted non-LIKE metacharacters. [#46875](https://github.com/ClickHouse/ClickHouse/pull/46875) ([Robert Schulze](https://github.com/rschu1ze)). + +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* More concise logging at trace level for PREWHERE steps [#46771](https://github.com/ClickHouse/ClickHouse/pull/46771) ([Alexander Gololobov](https://github.com/davenger)). +* Reduce updates of Mergeable Check [#46781](https://github.com/ClickHouse/ClickHouse/pull/46781) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). + diff --git a/docs/changelogs/v23.2.3.17-stable.md b/docs/changelogs/v23.2.3.17-stable.md new file mode 100644 index 00000000000..fb2c4e394dc --- /dev/null +++ b/docs/changelogs/v23.2.3.17-stable.md @@ -0,0 +1,23 @@ +--- +sidebar_position: 1 +sidebar_label: 2023 +--- + +# 2023 Changelog + +### ClickHouse release v23.2.3.17-stable (dec18bf7281) FIXME as compared to v23.2.2.20-stable (f6c269c8df2) + +#### Bug Fix (user-visible misbehavior in official stable or prestable release) + +* Backported in [#46907](https://github.com/ClickHouse/ClickHouse/issues/46907): - Fix incorrect alias recursion in QueryNormalizer. [#46609](https://github.com/ClickHouse/ClickHouse/pull/46609) ([Raúl Marín](https://github.com/Algunenano)). +* Backported in [#47091](https://github.com/ClickHouse/ClickHouse/issues/47091): - Fix arithmetic operations in aggregate optimization with `min` and `max`. [#46705](https://github.com/ClickHouse/ClickHouse/pull/46705) ([Duc Canh Le](https://github.com/canhld94)). +* Backported in [#46885](https://github.com/ClickHouse/ClickHouse/issues/46885): Fix MSan report in the `maxIntersections` function. This closes [#43126](https://github.com/ClickHouse/ClickHouse/issues/43126). [#46847](https://github.com/ClickHouse/ClickHouse/pull/46847) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#47067](https://github.com/ClickHouse/ClickHouse/issues/47067): Fix typo in systemd service, which causes the systemd service start to fail. [#47051](https://github.com/ClickHouse/ClickHouse/pull/47051) ([Palash Goel](https://github.com/palash-goel)). +* Backported in [#47259](https://github.com/ClickHouse/ClickHouse/issues/47259): Fix concrete columns PREWHERE support. [#47154](https://github.com/ClickHouse/ClickHouse/pull/47154) ([Azat Khuzhin](https://github.com/azat)). + +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Use /etc/default/clickhouse in systemd too [#47003](https://github.com/ClickHouse/ClickHouse/pull/47003) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* do flushUntrackedMemory when context switches [#47102](https://github.com/ClickHouse/ClickHouse/pull/47102) ([Sema Checherinda](https://github.com/CheSema)). +* Update typing for a new PyGithub version [#47123](https://github.com/ClickHouse/ClickHouse/pull/47123) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). + diff --git a/docs/changelogs/v23.2.4.12-stable.md b/docs/changelogs/v23.2.4.12-stable.md new file mode 100644 index 00000000000..2b6a689aee5 --- /dev/null +++ b/docs/changelogs/v23.2.4.12-stable.md @@ -0,0 +1,20 @@ +--- +sidebar_position: 1 +sidebar_label: 2023 +--- + +# 2023 Changelog + +### ClickHouse release v23.2.4.12-stable (8fe866cb035) FIXME as compared to v23.2.3.17-stable (dec18bf7281) + +#### Bug Fix (user-visible misbehavior in official stable or prestable release) + +* Backported in [#47277](https://github.com/ClickHouse/ClickHouse/issues/47277): Fix IPv4/IPv6 serialization/deserialization in binary formats that was broken in https://github.com/ClickHouse/ClickHouse/pull/43221. Closes [#46522](https://github.com/ClickHouse/ClickHouse/issues/46522). [#46616](https://github.com/ClickHouse/ClickHouse/pull/46616) ([Kruglov Pavel](https://github.com/Avogar)). +* Backported in [#47212](https://github.com/ClickHouse/ClickHouse/issues/47212): `INSERT` queries through native TCP protocol and HTTP protocol were not canceled correctly in some cases. It could lead to a partially applied query if a client canceled the query, or if a client died or, in rare cases, on network errors. As a result, it could lead to not working deduplication. Fixes [#27667](https://github.com/ClickHouse/ClickHouse/issues/27667) and [#45377](https://github.com/ClickHouse/ClickHouse/issues/45377). [#46681](https://github.com/ClickHouse/ClickHouse/pull/46681) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#47363](https://github.com/ClickHouse/ClickHouse/issues/47363): Fix possible deadlock on distributed query cancellation. [#47161](https://github.com/ClickHouse/ClickHouse/pull/47161) ([Kruglov Pavel](https://github.com/Avogar)). + +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Follow-up to [#46681](https://github.com/ClickHouse/ClickHouse/issues/46681) [#47284](https://github.com/ClickHouse/ClickHouse/pull/47284) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Add a manual trigger for release workflow [#47302](https://github.com/ClickHouse/ClickHouse/pull/47302) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). + diff --git a/docs/en/development/architecture.md b/docs/en/development/architecture.md index 50b338844df..dd46b294ddd 100644 --- a/docs/en/development/architecture.md +++ b/docs/en/development/architecture.md @@ -172,7 +172,7 @@ Global thread pool is `GlobalThreadPool` singleton class. To allocate thread fro Global pool is universal and all pools described below are implemented on top of it. This can be thought of as a hierarchy of pools. Any specialized pool takes its threads from the global pool using `ThreadPool` class. So the main purpose of any specialized pool is to apply limit on the number of simultaneous jobs and do job scheduling. If there are more jobs scheduled than threads in a pool, `ThreadPool` accumulates jobs in a queue with priorities. Each job has an integer priority. Default priority is zero. All jobs with higher priority values are started before any job with lower priority value. But there is no difference between already executing jobs, thus priority matters only when the pool in overloaded. -IO thread pool is implemented as a plain `ThreadPool` accessible via `IOThreadPool::get()` method. It is configured in the same way as global pool with `max_io_thread_pool_size`, `max_io_thread_pool_free_size` and `io_thread_pool_queue_size` settings. The main purpose of IO thread pool is to avoid exhaustion of the global pool with IO jobs, which could prevent queries from fully utilizing CPU. +IO thread pool is implemented as a plain `ThreadPool` accessible via `IOThreadPool::get()` method. It is configured in the same way as global pool with `max_io_thread_pool_size`, `max_io_thread_pool_free_size` and `io_thread_pool_queue_size` settings. The main purpose of IO thread pool is to avoid exhaustion of the global pool with IO jobs, which could prevent queries from fully utilizing CPU. Backup to S3 does significant amount of IO operations and to avoid impact on interactive queries there is a separate `BackupsIOThreadPool` configured with `max_backups_io_thread_pool_size`, `max_backups_io_thread_pool_free_size` and `backups_io_thread_pool_queue_size` settings. For periodic task execution there is `BackgroundSchedulePool` class. You can register tasks using `BackgroundSchedulePool::TaskHolder` objects and the pool ensures that no task runs two jobs at the same time. It also allows you to postpone task execution to a specific instant in the future or temporarily deactivate task. Global `Context` provides a few instances of this class for different purposes. For general purpose tasks `Context::getSchedulePool()` is used. diff --git a/docs/en/development/build-cross-s390x.md b/docs/en/development/build-cross-s390x.md new file mode 100644 index 00000000000..bfb94ef7ed0 --- /dev/null +++ b/docs/en/development/build-cross-s390x.md @@ -0,0 +1,123 @@ +--- +slug: /en/development/build-cross-s390x +sidebar_position: 69 +title: How to Build, Run and Debug ClickHouse on Linux for s390x (zLinux) +sidebar_label: Build on Linux for s390x (zLinux) +--- + +As of writing (2023/3/10) building for s390x considered to be experimental. Not all features can be enabled, has broken features and is currently under active development. + + +## Building + +As s390x does not support boringssl, it uses OpenSSL and has two related build options. +- By default, the s390x build will dynamically link to OpenSSL libraries. It will build OpenSSL shared objects, so it's not necessary to install OpenSSL beforehand. (This option is recommended in all cases.) +- Another option is to build OpenSSL in-tree. In this case two build flags need to be supplied to cmake +```bash +-DENABLE_OPENSSL_DYNAMIC=0 -DENABLE_OPENSSL=1 +``` + +These instructions assume that the host machine is x86_64 and has all the tooling required to build natively based on the [build instructions](../development/build.md). It also assumes that the host is Ubuntu 22.04 but the following instructions should also work on Ubuntu 20.04. + +In addition to installing the tooling used to build natively, the following additional packages need to be installed: + +```bash +apt-get install binutils-s390x-linux-gnu libc6-dev-s390x-cross gcc-s390x-linux-gnu binfmt-support qemu-user-static +``` + +If you wish to cross compile rust code install the rust cross compile target for s390x: +```bash +rustup target add s390x-unknown-linux-gnu +``` + +To build for s390x: +```bash +cmake -DCMAKE_TOOLCHAIN_FILE=cmake/linux/toolchain-s390x.cmake .. +ninja +``` + +## Running + +Once built, the binary can be run with, eg.: + +```bash +qemu-s390x-static -L /usr/s390x-linux-gnu ./clickhouse +``` + +## Debugging + +Install LLDB: + +```bash +apt-get install lldb-15 +``` + +To Debug a s390x executable, run clickhouse using QEMU in debug mode: + +```bash +qemu-s390x-static -g 31338 -L /usr/s390x-linux-gnu ./clickhouse +``` + +In another shell run LLDB and attach, replace `` and `` with the values corresponding to your environment. +```bash +lldb-15 +(lldb) target create ./clickhouse +Current executable set to '//ClickHouse//programs/clickhouse' (s390x). +(lldb) settings set target.source-map //ClickHouse +(lldb) gdb-remote 31338 +Process 1 stopped +* thread #1, stop reason = signal SIGTRAP + frame #0: 0x0000004020e74cd0 +-> 0x4020e74cd0: lgr %r2, %r15 + 0x4020e74cd4: aghi %r15, -160 + 0x4020e74cd8: xc 0(8,%r15), 0(%r15) + 0x4020e74cde: brasl %r14, 275429939040 +(lldb) b main +Breakpoint 1: 9 locations. +(lldb) c +Process 1 resuming +Process 1 stopped +* thread #1, stop reason = breakpoint 1.1 + frame #0: 0x0000004005cd9fc0 clickhouse`main(argc_=1, argv_=0x0000004020e594a8) at main.cpp:450:17 + 447 #if !defined(FUZZING_MODE) + 448 int main(int argc_, char ** argv_) + 449 { +-> 450 inside_main = true; + 451 SCOPE_EXIT({ inside_main = false; }); + 452 + 453 /// PHDR cache is required for query profiler to work reliably +``` + +## Visual Studio Code integration + +- (CodeLLDB extension)[https://github.com/vadimcn/vscode-lldb] is required for visual debugging, the (Command Variable)[https://github.com/rioj7/command-variable] extension can help dynamic launches if using (cmake variants)[https://github.com/microsoft/vscode-cmake-tools/blob/main/docs/variants.md]. +- Make sure to set the backend to your llvm installation eg. `"lldb.library": "/usr/lib/x86_64-linux-gnu/liblldb-15.so"` +- Launcher: +```json +{ + "version": "0.2.0", + "configurations": [ + { + "name": "Debug", + "type": "lldb", + "request": "custom", + "targetCreateCommands": ["target create ${command:cmake.launchTargetDirectory}/clickhouse"], + "processCreateCommands": ["settings set target.source-map ${input:targetdir} ${workspaceFolder}", "gdb-remote 31338"], + "sourceMap": { "${input:targetdir}": "${workspaceFolder}" }, + } + ], + "inputs": [ + { + "id": "targetdir", + "type": "command", + "command": "extension.commandvariable.transform", + "args": { + "text": "${command:cmake.launchTargetDirectory}", + "find": ".*/([^/]+)/[^/]+$", + "replace": "$1" + } + } + ] +} +``` +- Make sure to run the clickhouse executable in debug mode prior to launch. (It is also possible to create a `preLaunchTask` that automates this) \ No newline at end of file diff --git a/docs/en/development/contrib.md b/docs/en/development/contrib.md index 04158a0c3f7..4b296c43db4 100644 --- a/docs/en/development/contrib.md +++ b/docs/en/development/contrib.md @@ -1,6 +1,6 @@ --- slug: /en/development/contrib -sidebar_position: 71 +sidebar_position: 72 sidebar_label: Third-Party Libraries description: A list of third-party libraries used --- diff --git a/docs/en/development/developer-instruction.md b/docs/en/development/developer-instruction.md index b46cc10f99d..ace5ab79bb4 100644 --- a/docs/en/development/developer-instruction.md +++ b/docs/en/development/developer-instruction.md @@ -67,7 +67,7 @@ It generally means that the SSH keys for connecting to GitHub are missing. These You can also clone the repository via https protocol: - git clone --recursive--shallow-submodules https://github.com/ClickHouse/ClickHouse.git + git clone --recursive --shallow-submodules https://github.com/ClickHouse/ClickHouse.git This, however, will not let you send your changes to the server. You can still use it temporarily and add the SSH keys later replacing the remote address of the repository with `git remote` command. diff --git a/docs/en/development/style.md b/docs/en/development/style.md index 35683aa7822..78b1d1e9ebb 100644 --- a/docs/en/development/style.md +++ b/docs/en/development/style.md @@ -1,6 +1,6 @@ --- slug: /en/development/style -sidebar_position: 69 +sidebar_position: 70 sidebar_label: C++ Guide description: A list of recommendations regarding coding style, naming convention, formatting and more --- diff --git a/docs/en/development/tests.md b/docs/en/development/tests.md index 9ae49e8f707..eb424ee7cbe 100644 --- a/docs/en/development/tests.md +++ b/docs/en/development/tests.md @@ -1,6 +1,6 @@ --- slug: /en/development/tests -sidebar_position: 70 +sidebar_position: 71 sidebar_label: Testing title: ClickHouse Testing description: Most of ClickHouse features can be tested with functional tests and they are mandatory to use for every change in ClickHouse code that can be tested that way. @@ -31,6 +31,9 @@ folder and run the following command: PATH=$PATH: tests/clickhouse-test 01428_hash_set_nan_key ``` +Test results (`stderr` and `stdout`) are written to files `01428_hash_set_nan_key.[stderr|stdout]` which +are located near the test file itself (so for `queries/0_stateless/foo.sql` output will be in `queries/0_stateless/foo.stdout`). + For more options, see `tests/clickhouse-test --help`. You can simply run all tests or run subset of tests filtered by substring in test name: `./clickhouse-test substring`. There are also options to run tests in parallel or in randomized order. ### Adding a New Test diff --git a/docs/en/engines/table-engines/integrations/kafka.md b/docs/en/engines/table-engines/integrations/kafka.md index ef422632d3e..255ba06f056 100644 --- a/docs/en/engines/table-engines/integrations/kafka.md +++ b/docs/en/engines/table-engines/integrations/kafka.md @@ -19,8 +19,8 @@ Kafka lets you: ``` sql CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] ( - name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1], - name2 [type2] [DEFAULT|MATERIALIZED|ALIAS expr2], + name1 [type1], + name2 [type2], ... ) ENGINE = Kafka() SETTINGS @@ -113,6 +113,10 @@ Kafka(kafka_broker_list, kafka_topic_list, kafka_group_name, kafka_format +:::info +The Kafka table engine doesn't support columns with [default value](../../../sql-reference/statements/create/table.md#default_value). If you need columns with default value, you can add them at materialized view level (see below). +::: + ## Description {#description} The delivered messages are tracked automatically, so each message in a group is only counted once. If you want to get the data twice, then create a copy of the table with another group name. diff --git a/docs/en/engines/table-engines/mergetree-family/mergetree.md b/docs/en/engines/table-engines/mergetree-family/mergetree.md index 7c0416d3ea4..fc8060077b0 100644 --- a/docs/en/engines/table-engines/mergetree-family/mergetree.md +++ b/docs/en/engines/table-engines/mergetree-family/mergetree.md @@ -450,29 +450,32 @@ INDEX sample_index3 (lower(str), str) TYPE ngrambf_v1(3, 256, 2, 0) GRANULARITY Conditions in the `WHERE` clause contains calls of the functions that operate with columns. If the column is a part of an index, ClickHouse tries to use this index when performing the functions. ClickHouse supports different subsets of functions for using indexes. -The `set` index can be used with all functions. Function subsets for other indexes are shown in the table below. +Indexes of type `set` can be utilized by all functions. The other index types are supported as follows: | Function (operator) / Index | primary key | minmax | ngrambf_v1 | tokenbf_v1 | bloom_filter | -|------------------------------------------------------------------------------------------------------------|-------------|--------|-------------|-------------|---------------| -| [equals (=, ==)](/docs/en/sql-reference/functions/comparison-functions.md/#function-equals) | ✔ | ✔ | ✔ | ✔ | ✔ | -| [notEquals(!=, <>)](/docs/en/sql-reference/functions/comparison-functions.md/#function-notequals) | ✔ | ✔ | ✔ | ✔ | ✔ | -| [like](/docs/en/sql-reference/functions/string-search-functions.md/#function-like) | ✔ | ✔ | ✔ | ✔ | ✗ | -| [notLike](/docs/en/sql-reference/functions/string-search-functions.md/#function-notlike) | ✔ | ✔ | ✔ | ✔ | ✗ | -| [startsWith](/docs/en/sql-reference/functions/string-functions.md/#startswith) | ✔ | ✔ | ✔ | ✔ | ✗ | -| [endsWith](/docs/en/sql-reference/functions/string-functions.md/#endswith) | ✗ | ✗ | ✔ | ✔ | ✗ | -| [multiSearchAny](/docs/en/sql-reference/functions/string-search-functions.md/#function-multisearchany) | ✗ | ✗ | ✔ | ✗ | ✗ | -| [in](/docs/en/sql-reference/functions/in-functions#in-functions) | ✔ | ✔ | ✔ | ✔ | ✔ | -| [notIn](/docs/en/sql-reference/functions/in-functions#in-functions) | ✔ | ✔ | ✔ | ✔ | ✔ | -| [less (<)](/docs/en/sql-reference/functions/comparison-functions.md/#function-less) | ✔ | ✔ | ✗ | ✗ | ✗ | -| [greater (>)](/docs/en/sql-reference/functions/comparison-functions.md/#function-greater) | ✔ | ✔ | ✗ | ✗ | ✗ | -| [lessOrEquals (<=)](/docs/en/sql-reference/functions/comparison-functions.md/#function-lessorequals) | ✔ | ✔ | ✗ | ✗ | ✗ | -| [greaterOrEquals (>=)](/docs/en/sql-reference/functions/comparison-functions.md/#function-greaterorequals) | ✔ | ✔ | ✗ | ✗ | ✗ | -| [empty](/docs/en/sql-reference/functions/array-functions#function-empty) | ✔ | ✔ | ✗ | ✗ | ✗ | -| [notEmpty](/docs/en/sql-reference/functions/array-functions#function-notempty) | ✔ | ✔ | ✗ | ✗ | ✗ | -| hasToken | ✗ | ✗ | ✗ | ✔ | ✗ | -| hasTokenOrNull | ✗ | ✗ | ✗ | ✔ | ✗ | -| hasTokenCaseInsensitive | ✗ | ✗ | ✗ | ✔ | ✗ | -| hasTokenCaseInsensitiveOrNull | ✗ | ✗ | ✗ | ✔ | ✗ | +|------------------------------------------------------------------------------------------------------------|-------------|--------|------------|------------|--------------| +| [equals (=, ==)](/docs/en/sql-reference/functions/comparison-functions.md/#function-equals) | ✔ | ✔ | ✔ | ✔ | ✔ | +| [notEquals(!=, <>)](/docs/en/sql-reference/functions/comparison-functions.md/#function-notequals) | ✔ | ✔ | ✔ | ✔ | ✔ | +| [like](/docs/en/sql-reference/functions/string-search-functions.md/#function-like) | ✔ | ✔ | ✔ | ✔ | ✗ | +| [notLike](/docs/en/sql-reference/functions/string-search-functions.md/#function-notlike) | ✔ | ✔ | ✔ | ✔ | ✗ | +| [startsWith](/docs/en/sql-reference/functions/string-functions.md/#startswith) | ✔ | ✔ | ✔ | ✔ | ✗ | +| [endsWith](/docs/en/sql-reference/functions/string-functions.md/#endswith) | ✗ | ✗ | ✔ | ✔ | ✗ | +| [multiSearchAny](/docs/en/sql-reference/functions/string-search-functions.md/#function-multisearchany) | ✗ | ✗ | ✔ | ✗ | ✗ | +| [in](/docs/en/sql-reference/functions/in-functions#in-functions) | ✔ | ✔ | ✔ | ✔ | ✔ | +| [notIn](/docs/en/sql-reference/functions/in-functions#in-functions) | ✔ | ✔ | ✔ | ✔ | ✔ | +| [less (<)](/docs/en/sql-reference/functions/comparison-functions.md/#function-less) | ✔ | ✔ | ✗ | ✗ | ✗ | +| [greater (>)](/docs/en/sql-reference/functions/comparison-functions.md/#function-greater) | ✔ | ✔ | ✗ | ✗ | ✗ | +| [lessOrEquals (<=)](/docs/en/sql-reference/functions/comparison-functions.md/#function-lessorequals) | ✔ | ✔ | ✗ | ✗ | ✗ | +| [greaterOrEquals (>=)](/docs/en/sql-reference/functions/comparison-functions.md/#function-greaterorequals) | ✔ | ✔ | ✗ | ✗ | ✗ | +| [empty](/docs/en/sql-reference/functions/array-functions#function-empty) | ✔ | ✔ | ✗ | ✗ | ✗ | +| [notEmpty](/docs/en/sql-reference/functions/array-functions#function-notempty) | ✔ | ✔ | ✗ | ✗ | ✗ | +| [has](/docs/en/sql-reference/functions/array-functions#function-has) | ✗ | ✗ | ✔ | ✔ | ✔ | +| [hasAny](/docs/en/sql-reference/functions/array-functions#function-hasAny) | ✗ | ✗ | ✗ | ✗ | ✔ | +| [hasAll](/docs/en/sql-reference/functions/array-functions#function-hasAll) | ✗ | ✗ | ✗ | ✗ | ✔ | +| hasToken | ✗ | ✗ | ✗ | ✔ | ✗ | +| hasTokenOrNull | ✗ | ✗ | ✗ | ✔ | ✗ | +| hasTokenCaseInsensitive | ✗ | ✗ | ✗ | ✔ | ✗ | +| hasTokenCaseInsensitiveOrNull | ✗ | ✗ | ✗ | ✔ | ✗ | Functions with a constant argument that is less than ngram size can’t be used by `ngrambf_v1` for query optimization. diff --git a/docs/en/getting-started/example-datasets/nyc-taxi.md b/docs/en/getting-started/example-datasets/nyc-taxi.md index 69098f63037..9730faa873c 100644 --- a/docs/en/getting-started/example-datasets/nyc-taxi.md +++ b/docs/en/getting-started/example-datasets/nyc-taxi.md @@ -5,17 +5,19 @@ sidebar_position: 2 description: Data for billions of taxi and for-hire vehicle (Uber, Lyft, etc.) trips originating in New York City since 2009 --- +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + # New York Taxi Data The New York taxi data consists of 3+ billion taxi and for-hire vehicle (Uber, Lyft, etc.) trips originating in New York City since 2009. The dataset can be obtained in a couple of ways: -- insert the data directly into ClickHouse Cloud from S3 +- insert the data directly into ClickHouse Cloud from S3 or GCS - download prepared partitions -## Retrieve the Data from S3 +## Create the table trips -Let's grab a small subset of the data for getting familiar with it. The data is in TSV files in AWS S3, which is easily streamed into -ClickHouse Cloud using the `s3` table function. Start by creating a table for the taxi rides: +Start by creating a table for the taxi rides: ```sql CREATE TABLE trips ( @@ -38,9 +40,50 @@ CREATE TABLE trips ( dropoff_ntaname LowCardinality(String) ) ENGINE = MergeTree -PRIMARY KEY (pickup_datetime, dropoff_datetime) +PRIMARY KEY (pickup_datetime, dropoff_datetime); ``` +## Load the Data directly from Object Storage + +Let's grab a small subset of the data for getting familiar with it. The data is in TSV files in object storage, which is easily streamed into +ClickHouse Cloud using the `s3` table function. + +The same data is stored in both S3 and GCS; choose either tab. + + + + +The following command streams three files from a GCS bucket into the `trips` table (the `{0..2}` syntax is a wildcard for the values 0, 1, and 2): + +```sql +INSERT INTO trips +SELECT + trip_id, + pickup_datetime, + dropoff_datetime, + pickup_longitude, + pickup_latitude, + dropoff_longitude, + dropoff_latitude, + passenger_count, + trip_distance, + fare_amount, + extra, + tip_amount, + tolls_amount, + total_amount, + payment_type, + pickup_ntaname, + dropoff_ntaname +FROM s3( + 'https://storage.googleapis.com/clickhouse-public-datasets/nyc-taxi/trips_{0..2}.gz', + 'TabSeparatedWithNames' +); +``` + + + + The following command streams three files from an S3 bucket into the `trips` table (the `{0..2}` syntax is a wildcard for the values 0, 1, and 2): ```sql @@ -66,14 +109,19 @@ SELECT FROM s3( 'https://datasets-documentation.s3.eu-west-3.amazonaws.com/nyc-taxi/trips_{0..2}.gz', 'TabSeparatedWithNames' -) +); ``` + + + +## Sample Queries + Let's see how many rows were inserted: ```sql SELECT count() -FROM trips +FROM trips; ``` Each TSV file has about 1M rows, and the three files have 3,000,317 rows. Let's look at a few rows: @@ -81,7 +129,7 @@ Each TSV file has about 1M rows, and the three files have 3,000,317 rows. Let's ```sql SELECT * FROM trips -LIMIT 10 +LIMIT 10; ``` Notice there are columns for the pickup and dropoff dates, geo coordinates, fare details, New York neighborhoods, and more: @@ -110,7 +158,7 @@ SELECT FROM trips GROUP BY pickup_ntaname ORDER BY count DESC -LIMIT 10 +LIMIT 10; ``` The result is: @@ -137,7 +185,7 @@ SELECT passenger_count, avg(total_amount) FROM trips -GROUP BY passenger_count +GROUP BY passenger_count; ``` ```response @@ -165,7 +213,7 @@ SELECT count(*) FROM trips GROUP BY passenger_count, year, distance -ORDER BY year, count(*) DESC +ORDER BY year, count(*) DESC; ``` The first part of the result is: @@ -189,6 +237,10 @@ The first part of the result is: ## Download of Prepared Partitions {#download-of-prepared-partitions} +:::note +The following steps provide information about the original dataset, and a method for loading prepared partitions into a self-managed ClickHouse server environment. +::: + See https://github.com/toddwschneider/nyc-taxi-data and http://tech.marksblogg.com/billion-nyc-taxi-rides-redshift.html for the description of a dataset and instructions for downloading. Downloading will result in about 227 GB of uncompressed data in CSV files. The download takes about an hour over a 1 Gbit connection (parallel downloading from s3.amazonaws.com recovers at least half of a 1 Gbit channel). @@ -211,7 +263,7 @@ If you will run the queries described below, you have to use the full table name Q1: ``` sql -SELECT cab_type, count(*) FROM trips_mergetree GROUP BY cab_type +SELECT cab_type, count(*) FROM trips_mergetree GROUP BY cab_type; ``` 0.490 seconds. @@ -219,7 +271,7 @@ SELECT cab_type, count(*) FROM trips_mergetree GROUP BY cab_type Q2: ``` sql -SELECT passenger_count, avg(total_amount) FROM trips_mergetree GROUP BY passenger_count +SELECT passenger_count, avg(total_amount) FROM trips_mergetree GROUP BY passenger_count; ``` 1.224 seconds. @@ -227,7 +279,7 @@ SELECT passenger_count, avg(total_amount) FROM trips_mergetree GROUP BY passenge Q3: ``` sql -SELECT passenger_count, toYear(pickup_date) AS year, count(*) FROM trips_mergetree GROUP BY passenger_count, year +SELECT passenger_count, toYear(pickup_date) AS year, count(*) FROM trips_mergetree GROUP BY passenger_count, year; ``` 2.104 seconds. @@ -238,7 +290,7 @@ Q4: SELECT passenger_count, toYear(pickup_date) AS year, round(trip_distance) AS distance, count(*) FROM trips_mergetree GROUP BY passenger_count, year, distance -ORDER BY year, count(*) DESC +ORDER BY year, count(*) DESC; ``` 3.593 seconds. @@ -254,19 +306,19 @@ Creating a table on three servers: On each server: ``` sql -CREATE TABLE default.trips_mergetree_third ( trip_id UInt32, vendor_id Enum8('1' = 1, '2' = 2, 'CMT' = 3, 'VTS' = 4, 'DDS' = 5, 'B02512' = 10, 'B02598' = 11, 'B02617' = 12, 'B02682' = 13, 'B02764' = 14), pickup_date Date, pickup_datetime DateTime, dropoff_date Date, dropoff_datetime DateTime, store_and_fwd_flag UInt8, rate_code_id UInt8, pickup_longitude Float64, pickup_latitude Float64, dropoff_longitude Float64, dropoff_latitude Float64, passenger_count UInt8, trip_distance Float64, fare_amount Float32, extra Float32, mta_tax Float32, tip_amount Float32, tolls_amount Float32, ehail_fee Float32, improvement_surcharge Float32, total_amount Float32, payment_type_ Enum8('UNK' = 0, 'CSH' = 1, 'CRE' = 2, 'NOC' = 3, 'DIS' = 4), trip_type UInt8, pickup FixedString(25), dropoff FixedString(25), cab_type Enum8('yellow' = 1, 'green' = 2, 'uber' = 3), pickup_nyct2010_gid UInt8, pickup_ctlabel Float32, pickup_borocode UInt8, pickup_boroname Enum8('' = 0, 'Manhattan' = 1, 'Bronx' = 2, 'Brooklyn' = 3, 'Queens' = 4, 'Staten Island' = 5), pickup_ct2010 FixedString(6), pickup_boroct2010 FixedString(7), pickup_cdeligibil Enum8(' ' = 0, 'E' = 1, 'I' = 2), pickup_ntacode FixedString(4), pickup_ntaname Enum16('' = 0, 'Airport' = 1, 'Allerton-Pelham Gardens' = 2, 'Annadale-Huguenot-Prince\'s Bay-Eltingville' = 3, 'Arden Heights' = 4, 'Astoria' = 5, 'Auburndale' = 6, 'Baisley Park' = 7, 'Bath Beach' = 8, 'Battery Park City-Lower Manhattan' = 9, 'Bay Ridge' = 10, 'Bayside-Bayside Hills' = 11, 'Bedford' = 12, 'Bedford Park-Fordham North' = 13, 'Bellerose' = 14, 'Belmont' = 15, 'Bensonhurst East' = 16, 'Bensonhurst West' = 17, 'Borough Park' = 18, 'Breezy Point-Belle Harbor-Rockaway Park-Broad Channel' = 19, 'Briarwood-Jamaica Hills' = 20, 'Brighton Beach' = 21, 'Bronxdale' = 22, 'Brooklyn Heights-Cobble Hill' = 23, 'Brownsville' = 24, 'Bushwick North' = 25, 'Bushwick South' = 26, 'Cambria Heights' = 27, 'Canarsie' = 28, 'Carroll Gardens-Columbia Street-Red Hook' = 29, 'Central Harlem North-Polo Grounds' = 30, 'Central Harlem South' = 31, 'Charleston-Richmond Valley-Tottenville' = 32, 'Chinatown' = 33, 'Claremont-Bathgate' = 34, 'Clinton' = 35, 'Clinton Hill' = 36, 'Co-op City' = 37, 'College Point' = 38, 'Corona' = 39, 'Crotona Park East' = 40, 'Crown Heights North' = 41, 'Crown Heights South' = 42, 'Cypress Hills-City Line' = 43, 'DUMBO-Vinegar Hill-Downtown Brooklyn-Boerum Hill' = 44, 'Douglas Manor-Douglaston-Little Neck' = 45, 'Dyker Heights' = 46, 'East Concourse-Concourse Village' = 47, 'East Elmhurst' = 48, 'East Flatbush-Farragut' = 49, 'East Flushing' = 50, 'East Harlem North' = 51, 'East Harlem South' = 52, 'East New York' = 53, 'East New York (Pennsylvania Ave)' = 54, 'East Tremont' = 55, 'East Village' = 56, 'East Williamsburg' = 57, 'Eastchester-Edenwald-Baychester' = 58, 'Elmhurst' = 59, 'Elmhurst-Maspeth' = 60, 'Erasmus' = 61, 'Far Rockaway-Bayswater' = 62, 'Flatbush' = 63, 'Flatlands' = 64, 'Flushing' = 65, 'Fordham South' = 66, 'Forest Hills' = 67, 'Fort Greene' = 68, 'Fresh Meadows-Utopia' = 69, 'Ft. Totten-Bay Terrace-Clearview' = 70, 'Georgetown-Marine Park-Bergen Beach-Mill Basin' = 71, 'Glen Oaks-Floral Park-New Hyde Park' = 72, 'Glendale' = 73, 'Gramercy' = 74, 'Grasmere-Arrochar-Ft. Wadsworth' = 75, 'Gravesend' = 76, 'Great Kills' = 77, 'Greenpoint' = 78, 'Grymes Hill-Clifton-Fox Hills' = 79, 'Hamilton Heights' = 80, 'Hammels-Arverne-Edgemere' = 81, 'Highbridge' = 82, 'Hollis' = 83, 'Homecrest' = 84, 'Hudson Yards-Chelsea-Flatiron-Union Square' = 85, 'Hunters Point-Sunnyside-West Maspeth' = 86, 'Hunts Point' = 87, 'Jackson Heights' = 88, 'Jamaica' = 89, 'Jamaica Estates-Holliswood' = 90, 'Kensington-Ocean Parkway' = 91, 'Kew Gardens' = 92, 'Kew Gardens Hills' = 93, 'Kingsbridge Heights' = 94, 'Laurelton' = 95, 'Lenox Hill-Roosevelt Island' = 96, 'Lincoln Square' = 97, 'Lindenwood-Howard Beach' = 98, 'Longwood' = 99, 'Lower East Side' = 100, 'Madison' = 101, 'Manhattanville' = 102, 'Marble Hill-Inwood' = 103, 'Mariner\'s Harbor-Arlington-Port Ivory-Graniteville' = 104, 'Maspeth' = 105, 'Melrose South-Mott Haven North' = 106, 'Middle Village' = 107, 'Midtown-Midtown South' = 108, 'Midwood' = 109, 'Morningside Heights' = 110, 'Morrisania-Melrose' = 111, 'Mott Haven-Port Morris' = 112, 'Mount Hope' = 113, 'Murray Hill' = 114, 'Murray Hill-Kips Bay' = 115, 'New Brighton-Silver Lake' = 116, 'New Dorp-Midland Beach' = 117, 'New Springville-Bloomfield-Travis' = 118, 'North Corona' = 119, 'North Riverdale-Fieldston-Riverdale' = 120, 'North Side-South Side' = 121, 'Norwood' = 122, 'Oakland Gardens' = 123, 'Oakwood-Oakwood Beach' = 124, 'Ocean Hill' = 125, 'Ocean Parkway South' = 126, 'Old Astoria' = 127, 'Old Town-Dongan Hills-South Beach' = 128, 'Ozone Park' = 129, 'Park Slope-Gowanus' = 130, 'Parkchester' = 131, 'Pelham Bay-Country Club-City Island' = 132, 'Pelham Parkway' = 133, 'Pomonok-Flushing Heights-Hillcrest' = 134, 'Port Richmond' = 135, 'Prospect Heights' = 136, 'Prospect Lefferts Gardens-Wingate' = 137, 'Queens Village' = 138, 'Queensboro Hill' = 139, 'Queensbridge-Ravenswood-Long Island City' = 140, 'Rego Park' = 141, 'Richmond Hill' = 142, 'Ridgewood' = 143, 'Rikers Island' = 144, 'Rosedale' = 145, 'Rossville-Woodrow' = 146, 'Rugby-Remsen Village' = 147, 'Schuylerville-Throgs Neck-Edgewater Park' = 148, 'Seagate-Coney Island' = 149, 'Sheepshead Bay-Gerritsen Beach-Manhattan Beach' = 150, 'SoHo-TriBeCa-Civic Center-Little Italy' = 151, 'Soundview-Bruckner' = 152, 'Soundview-Castle Hill-Clason Point-Harding Park' = 153, 'South Jamaica' = 154, 'South Ozone Park' = 155, 'Springfield Gardens North' = 156, 'Springfield Gardens South-Brookville' = 157, 'Spuyten Duyvil-Kingsbridge' = 158, 'St. Albans' = 159, 'Stapleton-Rosebank' = 160, 'Starrett City' = 161, 'Steinway' = 162, 'Stuyvesant Heights' = 163, 'Stuyvesant Town-Cooper Village' = 164, 'Sunset Park East' = 165, 'Sunset Park West' = 166, 'Todt Hill-Emerson Hill-Heartland Village-Lighthouse Hill' = 167, 'Turtle Bay-East Midtown' = 168, 'University Heights-Morris Heights' = 169, 'Upper East Side-Carnegie Hill' = 170, 'Upper West Side' = 171, 'Van Cortlandt Village' = 172, 'Van Nest-Morris Park-Westchester Square' = 173, 'Washington Heights North' = 174, 'Washington Heights South' = 175, 'West Brighton' = 176, 'West Concourse' = 177, 'West Farms-Bronx River' = 178, 'West New Brighton-New Brighton-St. George' = 179, 'West Village' = 180, 'Westchester-Unionport' = 181, 'Westerleigh' = 182, 'Whitestone' = 183, 'Williamsbridge-Olinville' = 184, 'Williamsburg' = 185, 'Windsor Terrace' = 186, 'Woodhaven' = 187, 'Woodlawn-Wakefield' = 188, 'Woodside' = 189, 'Yorkville' = 190, 'park-cemetery-etc-Bronx' = 191, 'park-cemetery-etc-Brooklyn' = 192, 'park-cemetery-etc-Manhattan' = 193, 'park-cemetery-etc-Queens' = 194, 'park-cemetery-etc-Staten Island' = 195), pickup_puma UInt16, dropoff_nyct2010_gid UInt8, dropoff_ctlabel Float32, dropoff_borocode UInt8, dropoff_boroname Enum8('' = 0, 'Manhattan' = 1, 'Bronx' = 2, 'Brooklyn' = 3, 'Queens' = 4, 'Staten Island' = 5), dropoff_ct2010 FixedString(6), dropoff_boroct2010 FixedString(7), dropoff_cdeligibil Enum8(' ' = 0, 'E' = 1, 'I' = 2), dropoff_ntacode FixedString(4), dropoff_ntaname Enum16('' = 0, 'Airport' = 1, 'Allerton-Pelham Gardens' = 2, 'Annadale-Huguenot-Prince\'s Bay-Eltingville' = 3, 'Arden Heights' = 4, 'Astoria' = 5, 'Auburndale' = 6, 'Baisley Park' = 7, 'Bath Beach' = 8, 'Battery Park City-Lower Manhattan' = 9, 'Bay Ridge' = 10, 'Bayside-Bayside Hills' = 11, 'Bedford' = 12, 'Bedford Park-Fordham North' = 13, 'Bellerose' = 14, 'Belmont' = 15, 'Bensonhurst East' = 16, 'Bensonhurst West' = 17, 'Borough Park' = 18, 'Breezy Point-Belle Harbor-Rockaway Park-Broad Channel' = 19, 'Briarwood-Jamaica Hills' = 20, 'Brighton Beach' = 21, 'Bronxdale' = 22, 'Brooklyn Heights-Cobble Hill' = 23, 'Brownsville' = 24, 'Bushwick North' = 25, 'Bushwick South' = 26, 'Cambria Heights' = 27, 'Canarsie' = 28, 'Carroll Gardens-Columbia Street-Red Hook' = 29, 'Central Harlem North-Polo Grounds' = 30, 'Central Harlem South' = 31, 'Charleston-Richmond Valley-Tottenville' = 32, 'Chinatown' = 33, 'Claremont-Bathgate' = 34, 'Clinton' = 35, 'Clinton Hill' = 36, 'Co-op City' = 37, 'College Point' = 38, 'Corona' = 39, 'Crotona Park East' = 40, 'Crown Heights North' = 41, 'Crown Heights South' = 42, 'Cypress Hills-City Line' = 43, 'DUMBO-Vinegar Hill-Downtown Brooklyn-Boerum Hill' = 44, 'Douglas Manor-Douglaston-Little Neck' = 45, 'Dyker Heights' = 46, 'East Concourse-Concourse Village' = 47, 'East Elmhurst' = 48, 'East Flatbush-Farragut' = 49, 'East Flushing' = 50, 'East Harlem North' = 51, 'East Harlem South' = 52, 'East New York' = 53, 'East New York (Pennsylvania Ave)' = 54, 'East Tremont' = 55, 'East Village' = 56, 'East Williamsburg' = 57, 'Eastchester-Edenwald-Baychester' = 58, 'Elmhurst' = 59, 'Elmhurst-Maspeth' = 60, 'Erasmus' = 61, 'Far Rockaway-Bayswater' = 62, 'Flatbush' = 63, 'Flatlands' = 64, 'Flushing' = 65, 'Fordham South' = 66, 'Forest Hills' = 67, 'Fort Greene' = 68, 'Fresh Meadows-Utopia' = 69, 'Ft. Totten-Bay Terrace-Clearview' = 70, 'Georgetown-Marine Park-Bergen Beach-Mill Basin' = 71, 'Glen Oaks-Floral Park-New Hyde Park' = 72, 'Glendale' = 73, 'Gramercy' = 74, 'Grasmere-Arrochar-Ft. Wadsworth' = 75, 'Gravesend' = 76, 'Great Kills' = 77, 'Greenpoint' = 78, 'Grymes Hill-Clifton-Fox Hills' = 79, 'Hamilton Heights' = 80, 'Hammels-Arverne-Edgemere' = 81, 'Highbridge' = 82, 'Hollis' = 83, 'Homecrest' = 84, 'Hudson Yards-Chelsea-Flatiron-Union Square' = 85, 'Hunters Point-Sunnyside-West Maspeth' = 86, 'Hunts Point' = 87, 'Jackson Heights' = 88, 'Jamaica' = 89, 'Jamaica Estates-Holliswood' = 90, 'Kensington-Ocean Parkway' = 91, 'Kew Gardens' = 92, 'Kew Gardens Hills' = 93, 'Kingsbridge Heights' = 94, 'Laurelton' = 95, 'Lenox Hill-Roosevelt Island' = 96, 'Lincoln Square' = 97, 'Lindenwood-Howard Beach' = 98, 'Longwood' = 99, 'Lower East Side' = 100, 'Madison' = 101, 'Manhattanville' = 102, 'Marble Hill-Inwood' = 103, 'Mariner\'s Harbor-Arlington-Port Ivory-Graniteville' = 104, 'Maspeth' = 105, 'Melrose South-Mott Haven North' = 106, 'Middle Village' = 107, 'Midtown-Midtown South' = 108, 'Midwood' = 109, 'Morningside Heights' = 110, 'Morrisania-Melrose' = 111, 'Mott Haven-Port Morris' = 112, 'Mount Hope' = 113, 'Murray Hill' = 114, 'Murray Hill-Kips Bay' = 115, 'New Brighton-Silver Lake' = 116, 'New Dorp-Midland Beach' = 117, 'New Springville-Bloomfield-Travis' = 118, 'North Corona' = 119, 'North Riverdale-Fieldston-Riverdale' = 120, 'North Side-South Side' = 121, 'Norwood' = 122, 'Oakland Gardens' = 123, 'Oakwood-Oakwood Beach' = 124, 'Ocean Hill' = 125, 'Ocean Parkway South' = 126, 'Old Astoria' = 127, 'Old Town-Dongan Hills-South Beach' = 128, 'Ozone Park' = 129, 'Park Slope-Gowanus' = 130, 'Parkchester' = 131, 'Pelham Bay-Country Club-City Island' = 132, 'Pelham Parkway' = 133, 'Pomonok-Flushing Heights-Hillcrest' = 134, 'Port Richmond' = 135, 'Prospect Heights' = 136, 'Prospect Lefferts Gardens-Wingate' = 137, 'Queens Village' = 138, 'Queensboro Hill' = 139, 'Queensbridge-Ravenswood-Long Island City' = 140, 'Rego Park' = 141, 'Richmond Hill' = 142, 'Ridgewood' = 143, 'Rikers Island' = 144, 'Rosedale' = 145, 'Rossville-Woodrow' = 146, 'Rugby-Remsen Village' = 147, 'Schuylerville-Throgs Neck-Edgewater Park' = 148, 'Seagate-Coney Island' = 149, 'Sheepshead Bay-Gerritsen Beach-Manhattan Beach' = 150, 'SoHo-TriBeCa-Civic Center-Little Italy' = 151, 'Soundview-Bruckner' = 152, 'Soundview-Castle Hill-Clason Point-Harding Park' = 153, 'South Jamaica' = 154, 'South Ozone Park' = 155, 'Springfield Gardens North' = 156, 'Springfield Gardens South-Brookville' = 157, 'Spuyten Duyvil-Kingsbridge' = 158, 'St. Albans' = 159, 'Stapleton-Rosebank' = 160, 'Starrett City' = 161, 'Steinway' = 162, 'Stuyvesant Heights' = 163, 'Stuyvesant Town-Cooper Village' = 164, 'Sunset Park East' = 165, 'Sunset Park West' = 166, 'Todt Hill-Emerson Hill-Heartland Village-Lighthouse Hill' = 167, 'Turtle Bay-East Midtown' = 168, 'University Heights-Morris Heights' = 169, 'Upper East Side-Carnegie Hill' = 170, 'Upper West Side' = 171, 'Van Cortlandt Village' = 172, 'Van Nest-Morris Park-Westchester Square' = 173, 'Washington Heights North' = 174, 'Washington Heights South' = 175, 'West Brighton' = 176, 'West Concourse' = 177, 'West Farms-Bronx River' = 178, 'West New Brighton-New Brighton-St. George' = 179, 'West Village' = 180, 'Westchester-Unionport' = 181, 'Westerleigh' = 182, 'Whitestone' = 183, 'Williamsbridge-Olinville' = 184, 'Williamsburg' = 185, 'Windsor Terrace' = 186, 'Woodhaven' = 187, 'Woodlawn-Wakefield' = 188, 'Woodside' = 189, 'Yorkville' = 190, 'park-cemetery-etc-Bronx' = 191, 'park-cemetery-etc-Brooklyn' = 192, 'park-cemetery-etc-Manhattan' = 193, 'park-cemetery-etc-Queens' = 194, 'park-cemetery-etc-Staten Island' = 195), dropoff_puma UInt16) ENGINE = MergeTree(pickup_date, pickup_datetime, 8192) +CREATE TABLE default.trips_mergetree_third ( trip_id UInt32, vendor_id Enum8('1' = 1, '2' = 2, 'CMT' = 3, 'VTS' = 4, 'DDS' = 5, 'B02512' = 10, 'B02598' = 11, 'B02617' = 12, 'B02682' = 13, 'B02764' = 14), pickup_date Date, pickup_datetime DateTime, dropoff_date Date, dropoff_datetime DateTime, store_and_fwd_flag UInt8, rate_code_id UInt8, pickup_longitude Float64, pickup_latitude Float64, dropoff_longitude Float64, dropoff_latitude Float64, passenger_count UInt8, trip_distance Float64, fare_amount Float32, extra Float32, mta_tax Float32, tip_amount Float32, tolls_amount Float32, ehail_fee Float32, improvement_surcharge Float32, total_amount Float32, payment_type_ Enum8('UNK' = 0, 'CSH' = 1, 'CRE' = 2, 'NOC' = 3, 'DIS' = 4), trip_type UInt8, pickup FixedString(25), dropoff FixedString(25), cab_type Enum8('yellow' = 1, 'green' = 2, 'uber' = 3), pickup_nyct2010_gid UInt8, pickup_ctlabel Float32, pickup_borocode UInt8, pickup_boroname Enum8('' = 0, 'Manhattan' = 1, 'Bronx' = 2, 'Brooklyn' = 3, 'Queens' = 4, 'Staten Island' = 5), pickup_ct2010 FixedString(6), pickup_boroct2010 FixedString(7), pickup_cdeligibil Enum8(' ' = 0, 'E' = 1, 'I' = 2), pickup_ntacode FixedString(4), pickup_ntaname Enum16('' = 0, 'Airport' = 1, 'Allerton-Pelham Gardens' = 2, 'Annadale-Huguenot-Prince\'s Bay-Eltingville' = 3, 'Arden Heights' = 4, 'Astoria' = 5, 'Auburndale' = 6, 'Baisley Park' = 7, 'Bath Beach' = 8, 'Battery Park City-Lower Manhattan' = 9, 'Bay Ridge' = 10, 'Bayside-Bayside Hills' = 11, 'Bedford' = 12, 'Bedford Park-Fordham North' = 13, 'Bellerose' = 14, 'Belmont' = 15, 'Bensonhurst East' = 16, 'Bensonhurst West' = 17, 'Borough Park' = 18, 'Breezy Point-Belle Harbor-Rockaway Park-Broad Channel' = 19, 'Briarwood-Jamaica Hills' = 20, 'Brighton Beach' = 21, 'Bronxdale' = 22, 'Brooklyn Heights-Cobble Hill' = 23, 'Brownsville' = 24, 'Bushwick North' = 25, 'Bushwick South' = 26, 'Cambria Heights' = 27, 'Canarsie' = 28, 'Carroll Gardens-Columbia Street-Red Hook' = 29, 'Central Harlem North-Polo Grounds' = 30, 'Central Harlem South' = 31, 'Charleston-Richmond Valley-Tottenville' = 32, 'Chinatown' = 33, 'Claremont-Bathgate' = 34, 'Clinton' = 35, 'Clinton Hill' = 36, 'Co-op City' = 37, 'College Point' = 38, 'Corona' = 39, 'Crotona Park East' = 40, 'Crown Heights North' = 41, 'Crown Heights South' = 42, 'Cypress Hills-City Line' = 43, 'DUMBO-Vinegar Hill-Downtown Brooklyn-Boerum Hill' = 44, 'Douglas Manor-Douglaston-Little Neck' = 45, 'Dyker Heights' = 46, 'East Concourse-Concourse Village' = 47, 'East Elmhurst' = 48, 'East Flatbush-Farragut' = 49, 'East Flushing' = 50, 'East Harlem North' = 51, 'East Harlem South' = 52, 'East New York' = 53, 'East New York (Pennsylvania Ave)' = 54, 'East Tremont' = 55, 'East Village' = 56, 'East Williamsburg' = 57, 'Eastchester-Edenwald-Baychester' = 58, 'Elmhurst' = 59, 'Elmhurst-Maspeth' = 60, 'Erasmus' = 61, 'Far Rockaway-Bayswater' = 62, 'Flatbush' = 63, 'Flatlands' = 64, 'Flushing' = 65, 'Fordham South' = 66, 'Forest Hills' = 67, 'Fort Greene' = 68, 'Fresh Meadows-Utopia' = 69, 'Ft. Totten-Bay Terrace-Clearview' = 70, 'Georgetown-Marine Park-Bergen Beach-Mill Basin' = 71, 'Glen Oaks-Floral Park-New Hyde Park' = 72, 'Glendale' = 73, 'Gramercy' = 74, 'Grasmere-Arrochar-Ft. Wadsworth' = 75, 'Gravesend' = 76, 'Great Kills' = 77, 'Greenpoint' = 78, 'Grymes Hill-Clifton-Fox Hills' = 79, 'Hamilton Heights' = 80, 'Hammels-Arverne-Edgemere' = 81, 'Highbridge' = 82, 'Hollis' = 83, 'Homecrest' = 84, 'Hudson Yards-Chelsea-Flatiron-Union Square' = 85, 'Hunters Point-Sunnyside-West Maspeth' = 86, 'Hunts Point' = 87, 'Jackson Heights' = 88, 'Jamaica' = 89, 'Jamaica Estates-Holliswood' = 90, 'Kensington-Ocean Parkway' = 91, 'Kew Gardens' = 92, 'Kew Gardens Hills' = 93, 'Kingsbridge Heights' = 94, 'Laurelton' = 95, 'Lenox Hill-Roosevelt Island' = 96, 'Lincoln Square' = 97, 'Lindenwood-Howard Beach' = 98, 'Longwood' = 99, 'Lower East Side' = 100, 'Madison' = 101, 'Manhattanville' = 102, 'Marble Hill-Inwood' = 103, 'Mariner\'s Harbor-Arlington-Port Ivory-Graniteville' = 104, 'Maspeth' = 105, 'Melrose South-Mott Haven North' = 106, 'Middle Village' = 107, 'Midtown-Midtown South' = 108, 'Midwood' = 109, 'Morningside Heights' = 110, 'Morrisania-Melrose' = 111, 'Mott Haven-Port Morris' = 112, 'Mount Hope' = 113, 'Murray Hill' = 114, 'Murray Hill-Kips Bay' = 115, 'New Brighton-Silver Lake' = 116, 'New Dorp-Midland Beach' = 117, 'New Springville-Bloomfield-Travis' = 118, 'North Corona' = 119, 'North Riverdale-Fieldston-Riverdale' = 120, 'North Side-South Side' = 121, 'Norwood' = 122, 'Oakland Gardens' = 123, 'Oakwood-Oakwood Beach' = 124, 'Ocean Hill' = 125, 'Ocean Parkway South' = 126, 'Old Astoria' = 127, 'Old Town-Dongan Hills-South Beach' = 128, 'Ozone Park' = 129, 'Park Slope-Gowanus' = 130, 'Parkchester' = 131, 'Pelham Bay-Country Club-City Island' = 132, 'Pelham Parkway' = 133, 'Pomonok-Flushing Heights-Hillcrest' = 134, 'Port Richmond' = 135, 'Prospect Heights' = 136, 'Prospect Lefferts Gardens-Wingate' = 137, 'Queens Village' = 138, 'Queensboro Hill' = 139, 'Queensbridge-Ravenswood-Long Island City' = 140, 'Rego Park' = 141, 'Richmond Hill' = 142, 'Ridgewood' = 143, 'Rikers Island' = 144, 'Rosedale' = 145, 'Rossville-Woodrow' = 146, 'Rugby-Remsen Village' = 147, 'Schuylerville-Throgs Neck-Edgewater Park' = 148, 'Seagate-Coney Island' = 149, 'Sheepshead Bay-Gerritsen Beach-Manhattan Beach' = 150, 'SoHo-TriBeCa-Civic Center-Little Italy' = 151, 'Soundview-Bruckner' = 152, 'Soundview-Castle Hill-Clason Point-Harding Park' = 153, 'South Jamaica' = 154, 'South Ozone Park' = 155, 'Springfield Gardens North' = 156, 'Springfield Gardens South-Brookville' = 157, 'Spuyten Duyvil-Kingsbridge' = 158, 'St. Albans' = 159, 'Stapleton-Rosebank' = 160, 'Starrett City' = 161, 'Steinway' = 162, 'Stuyvesant Heights' = 163, 'Stuyvesant Town-Cooper Village' = 164, 'Sunset Park East' = 165, 'Sunset Park West' = 166, 'Todt Hill-Emerson Hill-Heartland Village-Lighthouse Hill' = 167, 'Turtle Bay-East Midtown' = 168, 'University Heights-Morris Heights' = 169, 'Upper East Side-Carnegie Hill' = 170, 'Upper West Side' = 171, 'Van Cortlandt Village' = 172, 'Van Nest-Morris Park-Westchester Square' = 173, 'Washington Heights North' = 174, 'Washington Heights South' = 175, 'West Brighton' = 176, 'West Concourse' = 177, 'West Farms-Bronx River' = 178, 'West New Brighton-New Brighton-St. George' = 179, 'West Village' = 180, 'Westchester-Unionport' = 181, 'Westerleigh' = 182, 'Whitestone' = 183, 'Williamsbridge-Olinville' = 184, 'Williamsburg' = 185, 'Windsor Terrace' = 186, 'Woodhaven' = 187, 'Woodlawn-Wakefield' = 188, 'Woodside' = 189, 'Yorkville' = 190, 'park-cemetery-etc-Bronx' = 191, 'park-cemetery-etc-Brooklyn' = 192, 'park-cemetery-etc-Manhattan' = 193, 'park-cemetery-etc-Queens' = 194, 'park-cemetery-etc-Staten Island' = 195), pickup_puma UInt16, dropoff_nyct2010_gid UInt8, dropoff_ctlabel Float32, dropoff_borocode UInt8, dropoff_boroname Enum8('' = 0, 'Manhattan' = 1, 'Bronx' = 2, 'Brooklyn' = 3, 'Queens' = 4, 'Staten Island' = 5), dropoff_ct2010 FixedString(6), dropoff_boroct2010 FixedString(7), dropoff_cdeligibil Enum8(' ' = 0, 'E' = 1, 'I' = 2), dropoff_ntacode FixedString(4), dropoff_ntaname Enum16('' = 0, 'Airport' = 1, 'Allerton-Pelham Gardens' = 2, 'Annadale-Huguenot-Prince\'s Bay-Eltingville' = 3, 'Arden Heights' = 4, 'Astoria' = 5, 'Auburndale' = 6, 'Baisley Park' = 7, 'Bath Beach' = 8, 'Battery Park City-Lower Manhattan' = 9, 'Bay Ridge' = 10, 'Bayside-Bayside Hills' = 11, 'Bedford' = 12, 'Bedford Park-Fordham North' = 13, 'Bellerose' = 14, 'Belmont' = 15, 'Bensonhurst East' = 16, 'Bensonhurst West' = 17, 'Borough Park' = 18, 'Breezy Point-Belle Harbor-Rockaway Park-Broad Channel' = 19, 'Briarwood-Jamaica Hills' = 20, 'Brighton Beach' = 21, 'Bronxdale' = 22, 'Brooklyn Heights-Cobble Hill' = 23, 'Brownsville' = 24, 'Bushwick North' = 25, 'Bushwick South' = 26, 'Cambria Heights' = 27, 'Canarsie' = 28, 'Carroll Gardens-Columbia Street-Red Hook' = 29, 'Central Harlem North-Polo Grounds' = 30, 'Central Harlem South' = 31, 'Charleston-Richmond Valley-Tottenville' = 32, 'Chinatown' = 33, 'Claremont-Bathgate' = 34, 'Clinton' = 35, 'Clinton Hill' = 36, 'Co-op City' = 37, 'College Point' = 38, 'Corona' = 39, 'Crotona Park East' = 40, 'Crown Heights North' = 41, 'Crown Heights South' = 42, 'Cypress Hills-City Line' = 43, 'DUMBO-Vinegar Hill-Downtown Brooklyn-Boerum Hill' = 44, 'Douglas Manor-Douglaston-Little Neck' = 45, 'Dyker Heights' = 46, 'East Concourse-Concourse Village' = 47, 'East Elmhurst' = 48, 'East Flatbush-Farragut' = 49, 'East Flushing' = 50, 'East Harlem North' = 51, 'East Harlem South' = 52, 'East New York' = 53, 'East New York (Pennsylvania Ave)' = 54, 'East Tremont' = 55, 'East Village' = 56, 'East Williamsburg' = 57, 'Eastchester-Edenwald-Baychester' = 58, 'Elmhurst' = 59, 'Elmhurst-Maspeth' = 60, 'Erasmus' = 61, 'Far Rockaway-Bayswater' = 62, 'Flatbush' = 63, 'Flatlands' = 64, 'Flushing' = 65, 'Fordham South' = 66, 'Forest Hills' = 67, 'Fort Greene' = 68, 'Fresh Meadows-Utopia' = 69, 'Ft. Totten-Bay Terrace-Clearview' = 70, 'Georgetown-Marine Park-Bergen Beach-Mill Basin' = 71, 'Glen Oaks-Floral Park-New Hyde Park' = 72, 'Glendale' = 73, 'Gramercy' = 74, 'Grasmere-Arrochar-Ft. Wadsworth' = 75, 'Gravesend' = 76, 'Great Kills' = 77, 'Greenpoint' = 78, 'Grymes Hill-Clifton-Fox Hills' = 79, 'Hamilton Heights' = 80, 'Hammels-Arverne-Edgemere' = 81, 'Highbridge' = 82, 'Hollis' = 83, 'Homecrest' = 84, 'Hudson Yards-Chelsea-Flatiron-Union Square' = 85, 'Hunters Point-Sunnyside-West Maspeth' = 86, 'Hunts Point' = 87, 'Jackson Heights' = 88, 'Jamaica' = 89, 'Jamaica Estates-Holliswood' = 90, 'Kensington-Ocean Parkway' = 91, 'Kew Gardens' = 92, 'Kew Gardens Hills' = 93, 'Kingsbridge Heights' = 94, 'Laurelton' = 95, 'Lenox Hill-Roosevelt Island' = 96, 'Lincoln Square' = 97, 'Lindenwood-Howard Beach' = 98, 'Longwood' = 99, 'Lower East Side' = 100, 'Madison' = 101, 'Manhattanville' = 102, 'Marble Hill-Inwood' = 103, 'Mariner\'s Harbor-Arlington-Port Ivory-Graniteville' = 104, 'Maspeth' = 105, 'Melrose South-Mott Haven North' = 106, 'Middle Village' = 107, 'Midtown-Midtown South' = 108, 'Midwood' = 109, 'Morningside Heights' = 110, 'Morrisania-Melrose' = 111, 'Mott Haven-Port Morris' = 112, 'Mount Hope' = 113, 'Murray Hill' = 114, 'Murray Hill-Kips Bay' = 115, 'New Brighton-Silver Lake' = 116, 'New Dorp-Midland Beach' = 117, 'New Springville-Bloomfield-Travis' = 118, 'North Corona' = 119, 'North Riverdale-Fieldston-Riverdale' = 120, 'North Side-South Side' = 121, 'Norwood' = 122, 'Oakland Gardens' = 123, 'Oakwood-Oakwood Beach' = 124, 'Ocean Hill' = 125, 'Ocean Parkway South' = 126, 'Old Astoria' = 127, 'Old Town-Dongan Hills-South Beach' = 128, 'Ozone Park' = 129, 'Park Slope-Gowanus' = 130, 'Parkchester' = 131, 'Pelham Bay-Country Club-City Island' = 132, 'Pelham Parkway' = 133, 'Pomonok-Flushing Heights-Hillcrest' = 134, 'Port Richmond' = 135, 'Prospect Heights' = 136, 'Prospect Lefferts Gardens-Wingate' = 137, 'Queens Village' = 138, 'Queensboro Hill' = 139, 'Queensbridge-Ravenswood-Long Island City' = 140, 'Rego Park' = 141, 'Richmond Hill' = 142, 'Ridgewood' = 143, 'Rikers Island' = 144, 'Rosedale' = 145, 'Rossville-Woodrow' = 146, 'Rugby-Remsen Village' = 147, 'Schuylerville-Throgs Neck-Edgewater Park' = 148, 'Seagate-Coney Island' = 149, 'Sheepshead Bay-Gerritsen Beach-Manhattan Beach' = 150, 'SoHo-TriBeCa-Civic Center-Little Italy' = 151, 'Soundview-Bruckner' = 152, 'Soundview-Castle Hill-Clason Point-Harding Park' = 153, 'South Jamaica' = 154, 'South Ozone Park' = 155, 'Springfield Gardens North' = 156, 'Springfield Gardens South-Brookville' = 157, 'Spuyten Duyvil-Kingsbridge' = 158, 'St. Albans' = 159, 'Stapleton-Rosebank' = 160, 'Starrett City' = 161, 'Steinway' = 162, 'Stuyvesant Heights' = 163, 'Stuyvesant Town-Cooper Village' = 164, 'Sunset Park East' = 165, 'Sunset Park West' = 166, 'Todt Hill-Emerson Hill-Heartland Village-Lighthouse Hill' = 167, 'Turtle Bay-East Midtown' = 168, 'University Heights-Morris Heights' = 169, 'Upper East Side-Carnegie Hill' = 170, 'Upper West Side' = 171, 'Van Cortlandt Village' = 172, 'Van Nest-Morris Park-Westchester Square' = 173, 'Washington Heights North' = 174, 'Washington Heights South' = 175, 'West Brighton' = 176, 'West Concourse' = 177, 'West Farms-Bronx River' = 178, 'West New Brighton-New Brighton-St. George' = 179, 'West Village' = 180, 'Westchester-Unionport' = 181, 'Westerleigh' = 182, 'Whitestone' = 183, 'Williamsbridge-Olinville' = 184, 'Williamsburg' = 185, 'Windsor Terrace' = 186, 'Woodhaven' = 187, 'Woodlawn-Wakefield' = 188, 'Woodside' = 189, 'Yorkville' = 190, 'park-cemetery-etc-Bronx' = 191, 'park-cemetery-etc-Brooklyn' = 192, 'park-cemetery-etc-Manhattan' = 193, 'park-cemetery-etc-Queens' = 194, 'park-cemetery-etc-Staten Island' = 195), dropoff_puma UInt16) ENGINE = MergeTree(pickup_date, pickup_datetime, 8192); ``` On the source server: ``` sql -CREATE TABLE trips_mergetree_x3 AS trips_mergetree_third ENGINE = Distributed(perftest, default, trips_mergetree_third, rand()) +CREATE TABLE trips_mergetree_x3 AS trips_mergetree_third ENGINE = Distributed(perftest, default, trips_mergetree_third, rand()); ``` The following query redistributes data: ``` sql -INSERT INTO trips_mergetree_x3 SELECT * FROM trips_mergetree +INSERT INTO trips_mergetree_x3 SELECT * FROM trips_mergetree; ``` This takes 2454 seconds. diff --git a/docs/en/interfaces/formats.md b/docs/en/interfaces/formats.md index b2b2c6d5b1e..db2e773a685 100644 --- a/docs/en/interfaces/formats.md +++ b/docs/en/interfaces/formats.md @@ -1232,50 +1232,52 @@ Each row is formatted as a single document and each column is formatted as a sin For output it uses the following correspondence between ClickHouse types and BSON types: -| ClickHouse type | BSON Type | -|-----------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------| -| [Bool](/docs/en/sql-reference/data-types/boolean.md) | `\x08` boolean | -| [Int8/UInt8](/docs/en/sql-reference/data-types/int-uint.md) | `\x10` int32 | -| [Int16UInt16](/docs/en/sql-reference/data-types/int-uint.md) | `\x10` int32 | -| [Int32](/docs/en/sql-reference/data-types/int-uint.md) | `\x10` int32 | -| [UInt32](/docs/en/sql-reference/data-types/int-uint.md) | `\x12` int64 | -| [Int64/UInt64](/docs/en/sql-reference/data-types/int-uint.md) | `\x12` int64 | -| [Float32/Float64](/docs/en/sql-reference/data-types/float.md) | `\x01` double | -| [Date](/docs/en/sql-reference/data-types/date.md)/[Date32](/docs/en/sql-reference/data-types/date32.md) | `\x10` int32 | -| [DateTime](/docs/en/sql-reference/data-types/datetime.md) | `\x12` int64 | -| [DateTime64](/docs/en/sql-reference/data-types/datetime64.md) | `\x09` datetime | -| [Decimal32](/docs/en/sql-reference/data-types/decimal.md) | `\x10` int32 | -| [Decimal64](/docs/en/sql-reference/data-types/decimal.md) | `\x12` int64 | -| [Decimal128](/docs/en/sql-reference/data-types/decimal.md) | `\x05` binary, `\x00` binary subtype, size = 16 | -| [Decimal256](/docs/en/sql-reference/data-types/decimal.md) | `\x05` binary, `\x00` binary subtype, size = 32 | -| [Int128/UInt128](/docs/en/sql-reference/data-types/int-uint.md) | `\x05` binary, `\x00` binary subtype, size = 16 | -| [Int256/UInt256](/docs/en/sql-reference/data-types/int-uint.md) | `\x05` binary, `\x00` binary subtype, size = 32 | +| ClickHouse type | BSON Type | +|-----------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------| +| [Bool](/docs/en/sql-reference/data-types/boolean.md) | `\x08` boolean | +| [Int8/UInt8](/docs/en/sql-reference/data-types/int-uint.md) | `\x10` int32 | +| [Int16UInt16](/docs/en/sql-reference/data-types/int-uint.md) | `\x10` int32 | +| [Int32](/docs/en/sql-reference/data-types/int-uint.md) | `\x10` int32 | +| [UInt32](/docs/en/sql-reference/data-types/int-uint.md) | `\x12` int64 | +| [Int64/UInt64](/docs/en/sql-reference/data-types/int-uint.md) | `\x12` int64 | +| [Float32/Float64](/docs/en/sql-reference/data-types/float.md) | `\x01` double | +| [Date](/docs/en/sql-reference/data-types/date.md)/[Date32](/docs/en/sql-reference/data-types/date32.md) | `\x10` int32 | +| [DateTime](/docs/en/sql-reference/data-types/datetime.md) | `\x12` int64 | +| [DateTime64](/docs/en/sql-reference/data-types/datetime64.md) | `\x09` datetime | +| [Decimal32](/docs/en/sql-reference/data-types/decimal.md) | `\x10` int32 | +| [Decimal64](/docs/en/sql-reference/data-types/decimal.md) | `\x12` int64 | +| [Decimal128](/docs/en/sql-reference/data-types/decimal.md) | `\x05` binary, `\x00` binary subtype, size = 16 | +| [Decimal256](/docs/en/sql-reference/data-types/decimal.md) | `\x05` binary, `\x00` binary subtype, size = 32 | +| [Int128/UInt128](/docs/en/sql-reference/data-types/int-uint.md) | `\x05` binary, `\x00` binary subtype, size = 16 | +| [Int256/UInt256](/docs/en/sql-reference/data-types/int-uint.md) | `\x05` binary, `\x00` binary subtype, size = 32 | | [String](/docs/en/sql-reference/data-types/string.md)/[FixedString](/docs/en/sql-reference/data-types/fixedstring.md) | `\x05` binary, `\x00` binary subtype or \x02 string if setting output_format_bson_string_as_string is enabled | -| [UUID](/docs/en/sql-reference/data-types/uuid.md) | `\x05` binary, `\x04` uuid subtype, size = 16 | -| [Array](/docs/en/sql-reference/data-types/array.md) | `\x04` array | -| [Tuple](/docs/en/sql-reference/data-types/tuple.md) | `\x04` array | -| [Named Tuple](/docs/en/sql-reference/data-types/tuple.md) | `\x03` document | -| [Map](/docs/en/sql-reference/data-types/map.md) (with String keys) | `\x03` document | +| [UUID](/docs/en/sql-reference/data-types/uuid.md) | `\x05` binary, `\x04` uuid subtype, size = 16 | +| [Array](/docs/en/sql-reference/data-types/array.md) | `\x04` array | +| [Tuple](/docs/en/sql-reference/data-types/tuple.md) | `\x04` array | +| [Named Tuple](/docs/en/sql-reference/data-types/tuple.md) | `\x03` document | +| [Map](/docs/en/sql-reference/data-types/map.md) (with String keys) | `\x03` document | +| [IPv4](/docs/en/sql-reference/data-types/domains/ipv4.md) | `\x10` int32 | +| [IPv6](/docs/en/sql-reference/data-types/domains/ipv6.md) | `\x05` binary, `\x00` binary subtype | For input it uses the following correspondence between BSON types and ClickHouse types: -| BSON Type | ClickHouse Type | -|------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `\x01` double | [Float32/Float64](/docs/en/sql-reference/data-types/float.md) | -| `\x02` string | [String](/docs/en/sql-reference/data-types/string.md)/[FixedString](/docs/en/sql-reference/data-types/fixedstring.md) | -| `\x03` document | [Map](/docs/en/sql-reference/data-types/map.md)/[Named Tuple](/docs/en/sql-reference/data-types/tuple.md) | -| `\x04` array | [Array](/docs/en/sql-reference/data-types/array.md)/[Tuple](/docs/en/sql-reference/data-types/tuple.md) | -| `\x05` binary, `\x00` binary subtype | [String](/docs/en/sql-reference/data-types/string.md)/[FixedString](/docs/en/sql-reference/data-types/fixedstring.md) | -| `\x05` binary, `\x02` old binary subtype | [String](/docs/en/sql-reference/data-types/string.md)/[FixedString](/docs/en/sql-reference/data-types/fixedstring.md) | -| `\x05` binary, `\x03` old uuid subtype | [UUID](/docs/en/sql-reference/data-types/uuid.md) | -| `\x05` binary, `\x04` uuid subtype | [UUID](/docs/en/sql-reference/data-types/uuid.md) | -| `\x07` ObjectId | [String](/docs/en/sql-reference/data-types/string.md)/[FixedString](/docs/en/sql-reference/data-types/fixedstring.md) | -| `\x08` boolean | [Bool](/docs/en/sql-reference/data-types/boolean.md) | -| `\x09` datetime | [DateTime64](/docs/en/sql-reference/data-types/datetime64.md) | -| `\x0A` null value | [NULL](/docs/en/sql-reference/data-types/nullable.md) | -| `\x0D` JavaScript code | [String](/docs/en/sql-reference/data-types/string.md)/[FixedString](/docs/en/sql-reference/data-types/fixedstring.md) | -| `\x0E` symbol | [String](/docs/en/sql-reference/data-types/string.md)/[FixedString](/docs/en/sql-reference/data-types/fixedstring.md) | -| `\x10` int32 | [Int32/UInt32](/docs/en/sql-reference/data-types/int-uint.md)/[Decimal32](/docs/en/sql-reference/data-types/decimal.md) | +| BSON Type | ClickHouse Type | +|------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `\x01` double | [Float32/Float64](/docs/en/sql-reference/data-types/float.md) | +| `\x02` string | [String](/docs/en/sql-reference/data-types/string.md)/[FixedString](/docs/en/sql-reference/data-types/fixedstring.md) | +| `\x03` document | [Map](/docs/en/sql-reference/data-types/map.md)/[Named Tuple](/docs/en/sql-reference/data-types/tuple.md) | +| `\x04` array | [Array](/docs/en/sql-reference/data-types/array.md)/[Tuple](/docs/en/sql-reference/data-types/tuple.md) | +| `\x05` binary, `\x00` binary subtype | [String](/docs/en/sql-reference/data-types/string.md)/[FixedString](/docs/en/sql-reference/data-types/fixedstring.md)/[IPv6](/docs/en/sql-reference/data-types/domains/ipv6.md) | +| `\x05` binary, `\x02` old binary subtype | [String](/docs/en/sql-reference/data-types/string.md)/[FixedString](/docs/en/sql-reference/data-types/fixedstring.md) | +| `\x05` binary, `\x03` old uuid subtype | [UUID](/docs/en/sql-reference/data-types/uuid.md) | +| `\x05` binary, `\x04` uuid subtype | [UUID](/docs/en/sql-reference/data-types/uuid.md) | +| `\x07` ObjectId | [String](/docs/en/sql-reference/data-types/string.md)/[FixedString](/docs/en/sql-reference/data-types/fixedstring.md) | +| `\x08` boolean | [Bool](/docs/en/sql-reference/data-types/boolean.md) | +| `\x09` datetime | [DateTime64](/docs/en/sql-reference/data-types/datetime64.md) | +| `\x0A` null value | [NULL](/docs/en/sql-reference/data-types/nullable.md) | +| `\x0D` JavaScript code | [String](/docs/en/sql-reference/data-types/string.md)/[FixedString](/docs/en/sql-reference/data-types/fixedstring.md) | +| `\x0E` symbol | [String](/docs/en/sql-reference/data-types/string.md)/[FixedString](/docs/en/sql-reference/data-types/fixedstring.md) | +| `\x10` int32 | [Int32/UInt32](/docs/en/sql-reference/data-types/int-uint.md)/[Decimal32](/docs/en/sql-reference/data-types/decimal.md)/[IPv4](/docs/en/sql-reference/data-types/domains/ipv4.md) | | `\x12` int64 | [Int64/UInt64](/docs/en/sql-reference/data-types/int-uint.md)/[Decimal64](/docs/en/sql-reference/data-types/decimal.md)/[DateTime64](/docs/en/sql-reference/data-types/datetime64.md) | Other BSON types are not supported. Also, it performs conversion between different integer types (for example, you can insert BSON int32 value into ClickHouse UInt8). @@ -1608,23 +1610,25 @@ See also [Format Schema](#formatschema). The table below shows supported data types and how they match ClickHouse [data types](/docs/en/sql-reference/data-types/index.md) in `INSERT` and `SELECT` queries. -| CapnProto data type (`INSERT`) | ClickHouse data type | CapnProto data type (`SELECT`) | -|--------------------------------|-----------------------------------------------------------|--------------------------------| -| `UINT8`, `BOOL` | [UInt8](/docs/en/sql-reference/data-types/int-uint.md) | `UINT8` | -| `INT8` | [Int8](/docs/en/sql-reference/data-types/int-uint.md) | `INT8` | -| `UINT16` | [UInt16](/docs/en/sql-reference/data-types/int-uint.md), [Date](/docs/en/sql-reference/data-types/date.md) | `UINT16` | -| `INT16` | [Int16](/docs/en/sql-reference/data-types/int-uint.md) | `INT16` | -| `UINT32` | [UInt32](/docs/en/sql-reference/data-types/int-uint.md), [DateTime](/docs/en/sql-reference/data-types/datetime.md) | `UINT32` | -| `INT32` | [Int32](/docs/en/sql-reference/data-types/int-uint.md) | `INT32` | -| `UINT64` | [UInt64](/docs/en/sql-reference/data-types/int-uint.md) | `UINT64` | -| `INT64` | [Int64](/docs/en/sql-reference/data-types/int-uint.md), [DateTime64](/docs/en/sql-reference/data-types/datetime.md) | `INT64` | -| `FLOAT32` | [Float32](/docs/en/sql-reference/data-types/float.md) | `FLOAT32` | -| `FLOAT64` | [Float64](/docs/en/sql-reference/data-types/float.md) | `FLOAT64` | -| `TEXT, DATA` | [String](/docs/en/sql-reference/data-types/string.md), [FixedString](/docs/en/sql-reference/data-types/fixedstring.md) | `TEXT, DATA` | -| `union(T, Void), union(Void, T)` | [Nullable(T)](/docs/en/sql-reference/data-types/date.md) | `union(T, Void), union(Void, T)` | -| `ENUM` | [Enum(8\|16)](/docs/en/sql-reference/data-types/enum.md) | `ENUM` | -| `LIST` | [Array](/docs/en/sql-reference/data-types/array.md) | `LIST` | -| `STRUCT` | [Tuple](/docs/en/sql-reference/data-types/tuple.md) | `STRUCT` | +| CapnProto data type (`INSERT`) | ClickHouse data type | CapnProto data type (`SELECT`) | +|----------------------------------|------------------------------------------------------------------------------------------------------------------------|------------------------------| +| `UINT8`, `BOOL` | [UInt8](/docs/en/sql-reference/data-types/int-uint.md) | `UINT8` | +| `INT8` | [Int8](/docs/en/sql-reference/data-types/int-uint.md) | `INT8` | +| `UINT16` | [UInt16](/docs/en/sql-reference/data-types/int-uint.md), [Date](/docs/en/sql-reference/data-types/date.md) | `UINT16` | +| `INT16` | [Int16](/docs/en/sql-reference/data-types/int-uint.md) | `INT16` | +| `UINT32` | [UInt32](/docs/en/sql-reference/data-types/int-uint.md), [DateTime](/docs/en/sql-reference/data-types/datetime.md) | `UINT32` | +| `INT32` | [Int32](/docs/en/sql-reference/data-types/int-uint.md) | `INT32` | +| `UINT64` | [UInt64](/docs/en/sql-reference/data-types/int-uint.md) | `UINT64` | +| `INT64` | [Int64](/docs/en/sql-reference/data-types/int-uint.md), [DateTime64](/docs/en/sql-reference/data-types/datetime.md) | `INT64` | +| `FLOAT32` | [Float32](/docs/en/sql-reference/data-types/float.md) | `FLOAT32` | +| `FLOAT64` | [Float64](/docs/en/sql-reference/data-types/float.md) | `FLOAT64` | +| `TEXT, DATA` | [String](/docs/en/sql-reference/data-types/string.md), [FixedString](/docs/en/sql-reference/data-types/fixedstring.md) | `TEXT, DATA` | +| `union(T, Void), union(Void, T)` | [Nullable(T)](/docs/en/sql-reference/data-types/date.md) | `union(T, Void), union(Void, T)` | +| `ENUM` | [Enum(8\ |16)](/docs/en/sql-reference/data-types/enum.md) | `ENUM` | +| `LIST` | [Array](/docs/en/sql-reference/data-types/array.md) | `LIST` | +| `STRUCT` | [Tuple](/docs/en/sql-reference/data-types/tuple.md) | `STRUCT` | +| `UINT32` | [IPv4](/docs/en/sql-reference/data-types/domains/ipv4.md) | `UINT32` | +| `DATA` | [IPv6](/docs/en/sql-reference/data-types/domains/ipv6.md) | `DATA` | For working with `Enum` in CapnProto format use the [format_capn_proto_enum_comparising_mode](/docs/en/operations/settings/settings-formats.md/#format_capn_proto_enum_comparising_mode) setting. @@ -1804,21 +1808,23 @@ ClickHouse Avro format supports reading and writing [Avro data files](https://av The table below shows supported data types and how they match ClickHouse [data types](/docs/en/sql-reference/data-types/index.md) in `INSERT` and `SELECT` queries. -| Avro data type `INSERT` | ClickHouse data type | Avro data type `SELECT` | -|---------------------------------------------|----------------------------------------------------------------------------------------------------|------------------------------| -| `boolean`, `int`, `long`, `float`, `double` | [Int(8\|16\|32)](/docs/en/sql-reference/data-types/int-uint.md), [UInt(8\|16\|32)](/docs/en/sql-reference/data-types/int-uint.md) | `int` | -| `boolean`, `int`, `long`, `float`, `double` | [Int64](/docs/en/sql-reference/data-types/int-uint.md), [UInt64](/docs/en/sql-reference/data-types/int-uint.md) | `long` | -| `boolean`, `int`, `long`, `float`, `double` | [Float32](/docs/en/sql-reference/data-types/float.md) | `float` | -| `boolean`, `int`, `long`, `float`, `double` | [Float64](/docs/en/sql-reference/data-types/float.md) | `double` | -| `bytes`, `string`, `fixed`, `enum` | [String](/docs/en/sql-reference/data-types/string.md) | `bytes` or `string` \* | -| `bytes`, `string`, `fixed` | [FixedString(N)](/docs/en/sql-reference/data-types/fixedstring.md) | `fixed(N)` | -| `enum` | [Enum(8\|16)](/docs/en/sql-reference/data-types/enum.md) | `enum` | -| `array(T)` | [Array(T)](/docs/en/sql-reference/data-types/array.md) | `array(T)` | -| `union(null, T)`, `union(T, null)` | [Nullable(T)](/docs/en/sql-reference/data-types/date.md) | `union(null, T)` | -| `null` | [Nullable(Nothing)](/docs/en/sql-reference/data-types/special-data-types/nothing.md) | `null` | -| `int (date)` \** | [Date](/docs/en/sql-reference/data-types/date.md) | `int (date)` \** | -| `long (timestamp-millis)` \** | [DateTime64(3)](/docs/en/sql-reference/data-types/datetime.md) | `long (timestamp-millis)` \* | -| `long (timestamp-micros)` \** | [DateTime64(6)](/docs/en/sql-reference/data-types/datetime.md) | `long (timestamp-micros)` \* | +| Avro data type `INSERT` | ClickHouse data type | Avro data type `SELECT` | +|---------------------------------------------|-----------------------------------------------------------------------------------------------------------------|-------------------------------------------------| +| `boolean`, `int`, `long`, `float`, `double` | [Int(8\ | 16\ |32)](/docs/en/sql-reference/data-types/int-uint.md), [UInt(8\|16\|32)](/docs/en/sql-reference/data-types/int-uint.md) | `int` | +| `boolean`, `int`, `long`, `float`, `double` | [Int64](/docs/en/sql-reference/data-types/int-uint.md), [UInt64](/docs/en/sql-reference/data-types/int-uint.md) | `long` | +| `boolean`, `int`, `long`, `float`, `double` | [Float32](/docs/en/sql-reference/data-types/float.md) | `float` | +| `boolean`, `int`, `long`, `float`, `double` | [Float64](/docs/en/sql-reference/data-types/float.md) | `double` | +| `bytes`, `string`, `fixed`, `enum` | [String](/docs/en/sql-reference/data-types/string.md) | `bytes` or `string` \* | +| `bytes`, `string`, `fixed` | [FixedString(N)](/docs/en/sql-reference/data-types/fixedstring.md) | `fixed(N)` | +| `enum` | [Enum(8\ | 16)](/docs/en/sql-reference/data-types/enum.md) | `enum` | +| `array(T)` | [Array(T)](/docs/en/sql-reference/data-types/array.md) | `array(T)` | +| `union(null, T)`, `union(T, null)` | [Nullable(T)](/docs/en/sql-reference/data-types/date.md) | `union(null, T)` | +| `null` | [Nullable(Nothing)](/docs/en/sql-reference/data-types/special-data-types/nothing.md) | `null` | +| `int (date)` \** | [Date](/docs/en/sql-reference/data-types/date.md) | `int (date)` \** | +| `long (timestamp-millis)` \** | [DateTime64(3)](/docs/en/sql-reference/data-types/datetime.md) | `long (timestamp-millis)` \* | +| `long (timestamp-micros)` \** | [DateTime64(6)](/docs/en/sql-reference/data-types/datetime.md) | `long (timestamp-micros)` \* | +| `int` | [IPv4](/docs/en/sql-reference/data-types/domains/ipv4.md) | `int` | +| `fixed(16)` | [IPv6](/docs/en/sql-reference/data-types/domains/ipv6.md) | `fixed(16)` | \* `bytes` is default, controlled by [output_format_avro_string_column_pattern](/docs/en/operations/settings/settings-formats.md/#output_format_avro_string_column_pattern) \** [Avro logical types](https://avro.apache.org/docs/current/spec.html#Logical+Types) @@ -1918,28 +1924,30 @@ Setting `format_avro_schema_registry_url` needs to be configured in `users.xml` The table below shows supported data types and how they match ClickHouse [data types](/docs/en/sql-reference/data-types/index.md) in `INSERT` and `SELECT` queries. -| Parquet data type (`INSERT`) | ClickHouse data type | Parquet data type (`SELECT`) | -|-----------------------------------------------|-----------------------------------------------------------------|------------------------------| -| `BOOL` | [Bool](/docs/en/sql-reference/data-types/boolean.md) | `BOOL` | -| `UINT8`, `BOOL` | [UInt8](/docs/en/sql-reference/data-types/int-uint.md) | `UINT8` | -| `INT8` | [Int8](/docs/en/sql-reference/data-types/int-uint.md) | `INT8` | -| `UINT16` | [UInt16](/docs/en/sql-reference/data-types/int-uint.md) | `UINT16` | -| `INT16` | [Int16](/docs/en/sql-reference/data-types/int-uint.md) | `INT16` | -| `UINT32` | [UInt32](/docs/en/sql-reference/data-types/int-uint.md) | `UINT32` | -| `INT32` | [Int32](/docs/en/sql-reference/data-types/int-uint.md) | `INT32` | -| `UINT64` | [UInt64](/docs/en/sql-reference/data-types/int-uint.md) | `UINT64` | -| `INT64` | [Int64](/docs/en/sql-reference/data-types/int-uint.md) | `INT64` | -| `FLOAT` | [Float32](/docs/en/sql-reference/data-types/float.md) | `FLOAT` | -| `DOUBLE` | [Float64](/docs/en/sql-reference/data-types/float.md) | `DOUBLE` | -| `DATE` | [Date32](/docs/en/sql-reference/data-types/date.md) | `DATE` | -| `TIME (ms)` | [DateTime](/docs/en/sql-reference/data-types/datetime.md) | `UINT32` | -| `TIMESTAMP`, `TIME (us, ns)` | [DateTime64](/docs/en/sql-reference/data-types/datetime64.md) | `TIMESTAMP` | -| `STRING`, `BINARY` | [String](/docs/en/sql-reference/data-types/string.md) | `BINARY` | -| `STRING`, `BINARY`, `FIXED_LENGTH_BYTE_ARRAY` | [FixedString](/docs/en/sql-reference/data-types/fixedstring.md) | `FIXED_LENGTH_BYTE_ARRAY` | -| `DECIMAL` | [Decimal](/docs/en/sql-reference/data-types/decimal.md) | `DECIMAL` | -| `LIST` | [Array](/docs/en/sql-reference/data-types/array.md) | `LIST` | -| `STRUCT` | [Tuple](/docs/en/sql-reference/data-types/tuple.md) | `STRUCT` | -| `MAP` | [Map](/docs/en/sql-reference/data-types/map.md) | `MAP` | +| Parquet data type (`INSERT`) | ClickHouse data type | Parquet data type (`SELECT`) | +|----------------------------------------------------|-----------------------------------------------------------------|------------------------------| +| `BOOL` | [Bool](/docs/en/sql-reference/data-types/boolean.md) | `BOOL` | +| `UINT8`, `BOOL` | [UInt8](/docs/en/sql-reference/data-types/int-uint.md) | `UINT8` | +| `INT8` | [Int8](/docs/en/sql-reference/data-types/int-uint.md) | `INT8` | +| `UINT16` | [UInt16](/docs/en/sql-reference/data-types/int-uint.md) | `UINT16` | +| `INT16` | [Int16](/docs/en/sql-reference/data-types/int-uint.md) | `INT16` | +| `UINT32` | [UInt32](/docs/en/sql-reference/data-types/int-uint.md) | `UINT32` | +| `INT32` | [Int32](/docs/en/sql-reference/data-types/int-uint.md) | `INT32` | +| `UINT64` | [UInt64](/docs/en/sql-reference/data-types/int-uint.md) | `UINT64` | +| `INT64` | [Int64](/docs/en/sql-reference/data-types/int-uint.md) | `INT64` | +| `FLOAT` | [Float32](/docs/en/sql-reference/data-types/float.md) | `FLOAT` | +| `DOUBLE` | [Float64](/docs/en/sql-reference/data-types/float.md) | `DOUBLE` | +| `DATE` | [Date32](/docs/en/sql-reference/data-types/date.md) | `DATE` | +| `TIME (ms)` | [DateTime](/docs/en/sql-reference/data-types/datetime.md) | `UINT32` | +| `TIMESTAMP`, `TIME (us, ns)` | [DateTime64](/docs/en/sql-reference/data-types/datetime64.md) | `TIMESTAMP` | +| `STRING`, `BINARY` | [String](/docs/en/sql-reference/data-types/string.md) | `BINARY` | +| `STRING`, `BINARY`, `FIXED_LENGTH_BYTE_ARRAY` | [FixedString](/docs/en/sql-reference/data-types/fixedstring.md) | `FIXED_LENGTH_BYTE_ARRAY` | +| `DECIMAL` | [Decimal](/docs/en/sql-reference/data-types/decimal.md) | `DECIMAL` | +| `LIST` | [Array](/docs/en/sql-reference/data-types/array.md) | `LIST` | +| `STRUCT` | [Tuple](/docs/en/sql-reference/data-types/tuple.md) | `STRUCT` | +| `MAP` | [Map](/docs/en/sql-reference/data-types/map.md) | `MAP` | +| `UINT32` | [IPv4](/docs/en/sql-reference/data-types/domains/ipv4.md) | `UINT32` | +| `FIXED_LENGTH_BYTE_ARRAY` | [IPv6](/docs/en/sql-reference/data-types/domains/ipv6.md) | `FIXED_LENGTH_BYTE_ARRAY` | Arrays can be nested and can have a value of the `Nullable` type as an argument. `Tuple` and `Map` types also can be nested. @@ -1973,6 +1981,7 @@ To exchange data with Hadoop, you can use [HDFS table engine](/docs/en/engines/t - [input_format_parquet_skip_columns_with_unsupported_types_in_schema_inference](/docs/en/operations/settings/settings-formats.md/#input_format_parquet_skip_columns_with_unsupported_types_in_schema_inference) - allow skipping columns with unsupported types while schema inference for Parquet format. Default value - `false`. - [output_format_parquet_fixed_string_as_fixed_byte_array](/docs/en/operations/settings/settings-formats.md/#output_format_parquet_fixed_string_as_fixed_byte_array) - use Parquet FIXED_LENGTH_BYTE_ARRAY type instead of Binary/String for FixedString columns. Default value - `true`. - [output_format_parquet_version](/docs/en/operations/settings/settings-formats.md/#output_format_parquet_version) - The version of Parquet format used in output format. Default value - `2.latest`. +- [output_format_parquet_compression_method](/docs/en/operations/settings/settings-formats.md/#output_format_parquet_compression_method) - compression method used in output Parquet format. Default value - `snappy`. ## Arrow {#data-format-arrow} @@ -2007,6 +2016,8 @@ The table below shows supported data types and how they match ClickHouse [data t | `LIST` | [Array](/docs/en/sql-reference/data-types/array.md) | `LIST` | | `STRUCT` | [Tuple](/docs/en/sql-reference/data-types/tuple.md) | `STRUCT` | | `MAP` | [Map](/docs/en/sql-reference/data-types/map.md) | `MAP` | +| `UINT32` | [IPv4](/docs/en/sql-reference/data-types/domains/ipv4.md) | `UINT32` | +| `FIXED_SIZE_BINARY`, `BINARY` | [IPv6](/docs/en/sql-reference/data-types/domains/ipv6.md) | `FIXED_SIZE_BINARY` | Arrays can be nested and can have a value of the `Nullable` type as an argument. `Tuple` and `Map` types also can be nested. @@ -2041,6 +2052,7 @@ $ clickhouse-client --query="SELECT * FROM {some_table} FORMAT Arrow" > {filenam - [input_format_arrow_allow_missing_columns](/docs/en/operations/settings/settings-formats.md/#input_format_arrow_allow_missing_columns) - allow missing columns while reading Arrow data. Default value - `false`. - [input_format_arrow_skip_columns_with_unsupported_types_in_schema_inference](/docs/en/operations/settings/settings-formats.md/#input_format_arrow_skip_columns_with_unsupported_types_in_schema_inference) - allow skipping columns with unsupported types while schema inference for Arrow format. Default value - `false`. - [output_format_arrow_fixed_string_as_fixed_byte_array](/docs/en/operations/settings/settings-formats.md/#output_format_arrow_fixed_string_as_fixed_byte_array) - use Arrow FIXED_SIZE_BINARY type instead of Binary/String for FixedString columns. Default value - `true`. +- [output_format_arrow_compression_method](/docs/en/operations/settings/settings-formats.md/#output_format_arrow_compression_method) - compression method used in output Arrow format. Default value - `none`. ## ArrowStream {#data-format-arrow-stream} @@ -2054,8 +2066,8 @@ $ clickhouse-client --query="SELECT * FROM {some_table} FORMAT Arrow" > {filenam The table below shows supported data types and how they match ClickHouse [data types](/docs/en/sql-reference/data-types/index.md) in `INSERT` and `SELECT` queries. -| ORC data type (`INSERT`) | ClickHouse data type | ORC data type (`SELECT`) | -|---------------------------------------|---------------------------------------------------------|--------------------------| +| ORC data type (`INSERT`) | ClickHouse data type | ORC data type (`SELECT`) | +|---------------------------------------|---------------------------------------------------------------|--------------------------| | `Boolean` | [UInt8](/docs/en/sql-reference/data-types/int-uint.md) | `Boolean` | | `Tinyint` | [Int8](/docs/en/sql-reference/data-types/int-uint.md) | `Tinyint` | | `Smallint` | [Int16](/docs/en/sql-reference/data-types/int-uint.md) | `Smallint` | @@ -2070,6 +2082,7 @@ The table below shows supported data types and how they match ClickHouse [data t | `List` | [Array](/docs/en/sql-reference/data-types/array.md) | `List` | | `Struct` | [Tuple](/docs/en/sql-reference/data-types/tuple.md) | `Struct` | | `Map` | [Map](/docs/en/sql-reference/data-types/map.md) | `Map` | +| `-` | [IPv4](/docs/en/sql-reference/data-types/int-uint.md) | `Int` | Other types are not supported. @@ -2096,6 +2109,7 @@ $ clickhouse-client --query="SELECT * FROM {some_table} FORMAT ORC" > {filename. ### Arrow format settings {#parquet-format-settings} - [output_format_arrow_string_as_string](/docs/en/operations/settings/settings-formats.md/#output_format_arrow_string_as_string) - use Arrow String type instead of Binary for String columns. Default value - `false`. +- [output_format_orc_compression_method](/docs/en/operations/settings/settings-formats.md/#output_format_orc_compression_method) - compression method used in output ORC format. Default value - `none`. - [input_format_arrow_import_nested](/docs/en/operations/settings/settings-formats.md/#input_format_arrow_import_nested) - allow inserting array of structs into Nested table in Arrow input format. Default value - `false`. - [input_format_arrow_case_insensitive_column_matching](/docs/en/operations/settings/settings-formats.md/#input_format_arrow_case_insensitive_column_matching) - ignore case when matching Arrow columns with ClickHouse columns. Default value - `false`. - [input_format_arrow_allow_missing_columns](/docs/en/operations/settings/settings-formats.md/#input_format_arrow_allow_missing_columns) - allow missing columns while reading Arrow data. Default value - `false`. @@ -2264,8 +2278,8 @@ ClickHouse supports reading and writing [MessagePack](https://msgpack.org/) data ### Data Types Matching {#data-types-matching-msgpack} -| MessagePack data type (`INSERT`) | ClickHouse data type | MessagePack data type (`SELECT`) | -|--------------------------------------------------------------------|-----------------------------------------------------------|------------------------------------| +| MessagePack data type (`INSERT`) | ClickHouse data type | MessagePack data type (`SELECT`) | +|--------------------------------------------------------------------|-----------------------------------------------------------------|------------------------------------| | `uint N`, `positive fixint` | [UIntN](/docs/en/sql-reference/data-types/int-uint.md) | `uint N` | | `int N`, `negative fixint` | [IntN](/docs/en/sql-reference/data-types/int-uint.md) | `int N` | | `bool` | [UInt8](/docs/en/sql-reference/data-types/int-uint.md) | `uint 8` | @@ -2278,6 +2292,8 @@ ClickHouse supports reading and writing [MessagePack](https://msgpack.org/) data | `uint 64` | [DateTime64](/docs/en/sql-reference/data-types/datetime.md) | `uint 64` | | `fixarray`, `array 16`, `array 32` | [Array](/docs/en/sql-reference/data-types/array.md) | `fixarray`, `array 16`, `array 32` | | `fixmap`, `map 16`, `map 32` | [Map](/docs/en/sql-reference/data-types/map.md) | `fixmap`, `map 16`, `map 32` | +| `uint 32` | [IPv4](/docs/en/sql-reference/data-types/domains/ipv4.md) | `uint 32` | +| `bin 8` | [String](/docs/en/sql-reference/data-types/string.md) | `bin 8` | Example: diff --git a/docs/en/interfaces/http.md b/docs/en/interfaces/http.md index 9af6df0c87d..32f323a63d5 100644 --- a/docs/en/interfaces/http.md +++ b/docs/en/interfaces/http.md @@ -309,6 +309,7 @@ The HTTP interface allows passing external data (external temporary tables) for ## Response Buffering {#response-buffering} You can enable response buffering on the server-side. The `buffer_size` and `wait_end_of_query` URL parameters are provided for this purpose. +Also settings `http_response_buffer_size` and `http_wait_end_of_query` can be used. `buffer_size` determines the number of bytes in the result to buffer in the server memory. If a result body is larger than this threshold, the buffer is written to the HTTP channel, and the remaining data is sent directly to the HTTP channel. diff --git a/docs/en/interfaces/schema-inference.md b/docs/en/interfaces/schema-inference.md index 25bdb0c36a3..e028b4a6d96 100644 --- a/docs/en/interfaces/schema-inference.md +++ b/docs/en/interfaces/schema-inference.md @@ -117,7 +117,7 @@ clickhouse-local --file='hobbies.jsonl' --table='hobbies' --query='SELECT * FROM 4 47 Brayan ['movies','skydiving'] ``` -# Using structure from insertion table {#using-structure-from-insertion-table} +## Using structure from insertion table {#using-structure-from-insertion-table} When table functions `file/s3/url/hdfs` are used to insert data into a table, there is an option to use the structure from the insertion table instead of extracting it from the data. @@ -222,7 +222,7 @@ INSERT INTO hobbies4 SELECT id, empty(hobbies) ? NULL : hobbies[1] FROM file(hob In this case, there are some operations performed on the column `hobbies` in the `SELECT` query to insert it into the table, so ClickHouse cannot use the structure from the insertion table, and schema inference will be used. -# Schema inference cache {#schema-inference-cache} +## Schema inference cache {#schema-inference-cache} For most input formats schema inference reads some data to determine its structure and this process can take some time. To prevent inferring the same schema every time ClickHouse read the data from the same file, the inferred schema is cached and when accessing the same file again, ClickHouse will use the schema from the cache. @@ -326,14 +326,14 @@ SELECT count() FROM system.schema_inference_cache WHERE storage='S3' └─────────┘ ``` -# Text formats {#text-formats} +## Text formats {#text-formats} For text formats, ClickHouse reads the data row by row, extracts column values according to the format, and then uses some recursive parsers and heuristics to determine the type for each value. The maximum number of rows read from the data in schema inference is controlled by the setting `input_format_max_rows_to_read_for_schema_inference` with default value 25000. By default, all inferred types are [Nullable](../sql-reference/data-types/nullable.md), but you can change this by setting `schema_inference_make_columns_nullable` (see examples in the [settings](#settings-for-text-formats) section). -## JSON formats {#json-formats} +### JSON formats {#json-formats} In JSON formats ClickHouse parses values according to the JSON specification and then tries to find the most appropriate data type for them. @@ -464,9 +464,9 @@ most likely this column contains only Nulls or empty Arrays/Maps. ... ``` -### JSON settings {#json-settings} +#### JSON settings {#json-settings} -#### input_format_json_read_objects_as_strings +##### input_format_json_read_objects_as_strings Enabling this setting allows reading nested JSON objects as strings. This setting can be used to read nested JSON objects without using JSON object type. @@ -486,7 +486,7 @@ DESC format(JSONEachRow, $$ └──────┴──────────────────┴──────────────┴────────────────────┴─────────┴──────────────────┴────────────────┘ ``` -#### input_format_json_try_infer_numbers_from_strings +##### input_format_json_try_infer_numbers_from_strings Enabling this setting allows inferring numbers from string values. @@ -507,7 +507,7 @@ DESC format(JSONEachRow, $$ └───────┴─────────────────┴──────────────┴────────────────────┴─────────┴──────────────────┴────────────────┘ ``` -#### input_format_json_read_numbers_as_strings +##### input_format_json_read_numbers_as_strings Enabling this setting allows reading numeric values as strings. @@ -528,7 +528,7 @@ DESC format(JSONEachRow, $$ └───────┴──────────────────┴──────────────┴────────────────────┴─────────┴──────────────────┴────────────────┘ ``` -#### input_format_json_read_bools_as_numbers +##### input_format_json_read_bools_as_numbers Enabling this setting allows reading Bool values as numbers. @@ -549,7 +549,7 @@ DESC format(JSONEachRow, $$ └───────┴─────────────────┴──────────────┴────────────────────┴─────────┴──────────────────┴────────────────┘ ``` -## CSV {#csv} +### CSV {#csv} In CSV format ClickHouse extracts column values from the row according to delimiters. ClickHouse expects all types except numbers and strings to be enclosed in double quotes. If the value is in double quotes, ClickHouse tries to parse the data inside quotes using the recursive parser and then tries to find the most appropriate data type for it. If the value is not in double quotes, ClickHouse tries to parse it as a number, @@ -726,7 +726,7 @@ $$) └──────────────┴───────────────┘ ``` -## TSV/TSKV {#tsv-tskv} +### TSV/TSKV {#tsv-tskv} In TSV/TSKV formats ClickHouse extracts column value from the row according to tabular delimiters and then parses extracted value using the recursive parser to determine the most appropriate type. If the type cannot be determined, ClickHouse treats this value as String. @@ -1019,7 +1019,7 @@ DESC format(TSV, '[1,2,3] 42.42 Hello World!') └──────┴──────────────────┴──────────────┴────────────────────┴─────────┴──────────────────┴────────────────┘ ``` -## CustomSeparated {#custom-separated} +### CustomSeparated {#custom-separated} In CustomSeparated format ClickHouse first extracts all column values from the row according to specified delimiters and then tries to infer the data type for each value according to escaping rule. @@ -1080,7 +1080,7 @@ $$) └────────┴───────────────┴────────────┘ ``` -## Template {#template} +### Template {#template} In Template format ClickHouse first extracts all column values from the row according to the specified template and then tries to infer the data type for each value according to its escaping rule. @@ -1120,7 +1120,7 @@ $$) └──────────┴────────────────────────┴──────────────┴────────────────────┴─────────┴──────────────────┴────────────────┘ ``` -## Regexp {#regexp} +### Regexp {#regexp} Similar to Template, in Regexp format ClickHouse first extracts all column values from the row according to specified regular expression and then tries to infer data type for each value according to the specified escaping rule. @@ -1142,9 +1142,9 @@ Line: value_1=2, value_2="Some string 2", value_3="[4, 5, NULL]"$$) └──────┴────────────────────────┴──────────────┴────────────────────┴─────────┴──────────────────┴────────────────┘ ``` -## Settings for text formats {settings-for-text-formats} +### Settings for text formats {#settings-for-text-formats} -### input_format_max_rows_to_read_for_schema_inference +#### input_format_max_rows_to_read_for_schema_inference This setting controls the maximum number of rows to be read while schema inference. The more rows are read, the more time is spent on schema inference, but the greater the chance to @@ -1152,7 +1152,7 @@ correctly determine the types (especially when the data contains a lot of nulls) Default value: `25000`. -### column_names_for_schema_inference +#### column_names_for_schema_inference The list of column names to use in schema inference for formats without explicit column names. Specified names will be used instead of default `c1,c2,c3,...`. The format: `column1,column2,column3,...`. @@ -1169,7 +1169,7 @@ DESC format(TSV, 'Hello, World! 42 [1, 2, 3]') settings column_names_for_schema_ └──────┴────────────────────────┴──────────────┴────────────────────┴─────────┴──────────────────┴────────────────┘ ``` -### schema_inference_hints +#### schema_inference_hints The list of column names and types to use in schema inference instead of automatically determined types. The format: 'column_name1 column_type1, column_name2 column_type2, ...'. This setting can be used to specify the types of columns that could not be determined automatically or for optimizing the schema. @@ -1189,7 +1189,7 @@ DESC format(JSONEachRow, '{"id" : 1, "age" : 25, "name" : "Josh", "status" : nul └─────────┴─────────────────────────┴──────────────┴────────────────────┴─────────┴──────────────────┴────────────────┘ ``` -### schema_inference_make_columns_nullable +#### schema_inference_make_columns_nullable Controls making inferred types `Nullable` in schema inference for formats without information about nullability. If the setting is enabled, all inferred type will be `Nullable`, if disabled, the inferred type will be `Nullable` only if the column contains `NULL` in a sample that is parsed during schema inference. @@ -1232,7 +1232,7 @@ DESC format(JSONEachRow, $$ └─────────┴──────────────────┴──────────────┴────────────────────┴─────────┴──────────────────┴────────────────┘ ``` -### input_format_try_infer_integers +#### input_format_try_infer_integers If enabled, ClickHouse will try to infer integers instead of floats in schema inference for text formats. If all numbers in the column from sample data are integers, the result type will be `Int64`, if at least one number is float, the result type will be `Float64`. @@ -1289,7 +1289,7 @@ DESC format(JSONEachRow, $$ └────────┴───────────────────┴──────────────┴────────────────────┴─────────┴──────────────────┴────────────────┘ ``` -### input_format_try_infer_datetimes +#### input_format_try_infer_datetimes If enabled, ClickHouse will try to infer type `DateTime64` from string fields in schema inference for text formats. If all fields from a column in sample data were successfully parsed as datetimes, the result type will be `DateTime64(9)`, @@ -1337,7 +1337,7 @@ DESC format(JSONEachRow, $$ Note: Parsing datetimes during schema inference respect setting [date_time_input_format](/docs/en/operations/settings/settings-formats.md#date_time_input_format) -### input_format_try_infer_dates +#### input_format_try_infer_dates If enabled, ClickHouse will try to infer type `Date` from string fields in schema inference for text formats. If all fields from a column in sample data were successfully parsed as dates, the result type will be `Date`, @@ -1383,14 +1383,14 @@ DESC format(JSONEachRow, $$ └──────┴──────────────────┴──────────────┴────────────────────┴─────────┴──────────────────┴────────────────┘ ``` -# Self describing formats {#self-describing-formats} +## Self describing formats {#self-describing-formats} Self-describing formats contain information about the structure of the data in the data itself, it can be some header with a description, a binary type tree, or some kind of table. To automatically infer a schema from files in such formats, ClickHouse reads a part of the data containing information about the types and converts it into a schema of the ClickHouse table. -## Formats with -WithNamesAndTypes suffix {#formats-with-names-and-types} +### Formats with -WithNamesAndTypes suffix {#formats-with-names-and-types} ClickHouse supports some text formats with the suffix -WithNamesAndTypes. This suffix means that the data contains two additional rows with column names and types before the actual data. While schema inference for such formats, ClickHouse reads the first two rows and extracts column names and types. @@ -1412,7 +1412,7 @@ $$) └──────┴──────────────┴──────────────┴────────────────────┴─────────┴──────────────────┴────────────────┘ ``` -## JSON formats with metadata {#json-with-metadata} +### JSON formats with metadata {#json-with-metadata} Some JSON input formats ([JSON](formats.md#json), [JSONCompact](formats.md#json-compact), [JSONColumnsWithMetadata](formats.md#jsoncolumnswithmetadata)) contain metadata with column names and types. In schema inference for such formats, ClickHouse reads this metadata. @@ -1465,7 +1465,7 @@ $$) └──────┴──────────────┴──────────────┴────────────────────┴─────────┴──────────────────┴────────────────┘ ``` -## Avro {#avro} +### Avro {#avro} In Avro format ClickHouse reads its schema from the data and converts it to ClickHouse schema using the following type matches: @@ -1485,7 +1485,7 @@ In Avro format ClickHouse reads its schema from the data and converts it to Clic Other Avro types are not supported. -## Parquet {#parquet} +### Parquet {#parquet} In Parquet format ClickHouse reads its schema from the data and converts it to ClickHouse schema using the following type matches: @@ -1513,7 +1513,7 @@ In Parquet format ClickHouse reads its schema from the data and converts it to C Other Parquet types are not supported. By default, all inferred types are inside `Nullable`, but it can be changed using the setting `schema_inference_make_columns_nullable`. -## Arrow {#arrow} +### Arrow {#arrow} In Arrow format ClickHouse reads its schema from the data and converts it to ClickHouse schema using the following type matches: @@ -1541,7 +1541,7 @@ In Arrow format ClickHouse reads its schema from the data and converts it to Cli Other Arrow types are not supported. By default, all inferred types are inside `Nullable`, but it can be changed using the setting `schema_inference_make_columns_nullable`. -## ORC {#orc} +### ORC {#orc} In ORC format ClickHouse reads its schema from the data and converts it to ClickHouse schema using the following type matches: @@ -1564,17 +1564,17 @@ In ORC format ClickHouse reads its schema from the data and converts it to Click Other ORC types are not supported. By default, all inferred types are inside `Nullable`, but it can be changed using the setting `schema_inference_make_columns_nullable`. -## Native {#native} +### Native {#native} Native format is used inside ClickHouse and contains the schema in the data. In schema inference, ClickHouse reads the schema from the data without any transformations. -# Formats with external schema {#formats-with-external-schema} +## Formats with external schema {#formats-with-external-schema} Such formats require a schema describing the data in a separate file in a specific schema language. To automatically infer a schema from files in such formats, ClickHouse reads external schema from a separate file and transforms it to a ClickHouse table schema. -# Protobuf {#protobuf} +### Protobuf {#protobuf} In schema inference for Protobuf format ClickHouse uses the following type matches: @@ -1592,7 +1592,7 @@ In schema inference for Protobuf format ClickHouse uses the following type match | `repeated T` | [Array(T)](../sql-reference/data-types/array.md) | | `message`, `group` | [Tuple](../sql-reference/data-types/tuple.md) | -# CapnProto {#capnproto} +### CapnProto {#capnproto} In schema inference for CapnProto format ClickHouse uses the following type matches: @@ -1615,13 +1615,13 @@ In schema inference for CapnProto format ClickHouse uses the following type matc | `struct` | [Tuple](../sql-reference/data-types/tuple.md) | | `union(T, Void)`, `union(Void, T)` | [Nullable(T)](../sql-reference/data-types/nullable.md) | -# Strong-typed binary formats {#strong-typed-binary-formats} +## Strong-typed binary formats {#strong-typed-binary-formats} In such formats, each serialized value contains information about its type (and possibly about its name), but there is no information about the whole table. In schema inference for such formats, ClickHouse reads data row by row (up to `input_format_max_rows_to_read_for_schema_inference` rows) and extracts the type (and possibly name) for each value from the data and then converts these types to ClickHouse types. -## MsgPack {msgpack} +### MsgPack {#msgpack} In MsgPack format there is no delimiter between rows, to use schema inference for this format you should specify the number of columns in the table using the setting `input_format_msgpack_number_of_columns`. ClickHouse uses the following type matches: @@ -1641,7 +1641,7 @@ using the setting `input_format_msgpack_number_of_columns`. ClickHouse uses the By default, all inferred types are inside `Nullable`, but it can be changed using the setting `schema_inference_make_columns_nullable`. -## BSONEachRow {#bsoneachrow} +### BSONEachRow {#bsoneachrow} In BSONEachRow each row of data is presented as a BSON document. In schema inference ClickHouse reads BSON documents one by one and extracts values, names, and types from the data and then transforms these types to ClickHouse types using the following type matches: @@ -1661,11 +1661,11 @@ values, names, and types from the data and then transforms these types to ClickH By default, all inferred types are inside `Nullable`, but it can be changed using the setting `schema_inference_make_columns_nullable`. -# Formats with constant schema {#formats-with-constant-schema} +## Formats with constant schema {#formats-with-constant-schema} Data in such formats always have the same schema. -## LineAsString {#line-as-string} +### LineAsString {#line-as-string} In this format, ClickHouse reads the whole line from the data into a single column with `String` data type. The inferred type for this format is always `String` and the column name is `line`. @@ -1680,7 +1680,7 @@ DESC format(LineAsString, 'Hello\nworld!') └──────┴────────┴──────────────┴────────────────────┴─────────┴──────────────────┴────────────────┘ ``` -## JSONAsString {#json-as-string} +### JSONAsString {#json-as-string} In this format, ClickHouse reads the whole JSON object from the data into a single column with `String` data type. The inferred type for this format is always `String` and the column name is `json`. @@ -1695,7 +1695,7 @@ DESC format(JSONAsString, '{"x" : 42, "y" : "Hello, World!"}') └──────┴────────┴──────────────┴────────────────────┴─────────┴──────────────────┴────────────────┘ ``` -## JSONAsObject {#json-as-object} +### JSONAsObject {#json-as-object} In this format, ClickHouse reads the whole JSON object from the data into a single column with `Object('json')` data type. Inferred type for this format is always `String` and the column name is `json`. diff --git a/docs/en/operations/server-configuration-parameters/settings.md b/docs/en/operations/server-configuration-parameters/settings.md index 17d03dfa4ec..0424c3520e0 100644 --- a/docs/en/operations/server-configuration-parameters/settings.md +++ b/docs/en/operations/server-configuration-parameters/settings.md @@ -765,7 +765,7 @@ Default value: `0`. ## concurrent_threads_soft_limit_ratio_to_cores {#concurrent_threads_soft_limit_ratio_to_cores} The maximum number of query processing threads as multiple of number of logical cores. -More details: [concurrent_threads_soft_limit_num](#concurrent-threads-soft-limit-num). +More details: [concurrent_threads_soft_limit_num](#concurrent_threads_soft_limit_num). Possible values: @@ -967,6 +967,7 @@ The maximum number of jobs that can be scheduled on the Global Thread pool. Incr Possible values: - Positive integer. +- 0 — No limit. Default value: `10000`. @@ -976,6 +977,69 @@ Default value: `10000`. 12000 ``` +## max_io_thread_pool_size {#max-io-thread-pool-size} + +ClickHouse uses threads from the IO Thread pool to do some IO operations (e.g. to interact with S3). `max_io_thread_pool_size` limits the maximum number of threads in the pool. + +Possible values: + +- Positive integer. + +Default value: `100`. + +## max_io_thread_pool_free_size {#max-io-thread-pool-free-size} + +If the number of **idle** threads in the IO Thread pool exceeds `max_io_thread_pool_free_size`, ClickHouse will release resources occupied by idling threads and decrease the pool size. Threads can be created again if necessary. + +Possible values: + +- Positive integer. + +Default value: `0`. + +## io_thread_pool_queue_size {#io-thread-pool-queue-size} + +The maximum number of jobs that can be scheduled on the IO Thread pool. + +Possible values: + +- Positive integer. +- 0 — No limit. + +Default value: `10000`. + +## max_backups_io_thread_pool_size {#max-backups-io-thread-pool-size} + +ClickHouse uses threads from the Backups IO Thread pool to do S3 backup IO operations. `max_backups_io_thread_pool_size` limits the maximum number of threads in the pool. + +Possible values: + +- Positive integer. + +Default value: `1000`. + +## max_backups_io_thread_pool_free_size {#max-backups-io-thread-pool-free-size} + +If the number of **idle** threads in the Backups IO Thread pool exceeds `max_backup_io_thread_pool_free_size`, ClickHouse will release resources occupied by idling threads and decrease the pool size. Threads can be created again if necessary. + +Possible values: + +- Positive integer. +- Zero. + +Default value: `0`. + +## backups_io_thread_pool_queue_size {#backups-io-thread-pool-queue-size} + +The maximum number of jobs that can be scheduled on the Backups IO Thread pool. It is recommended to keep this queue unlimited due to the current S3 backup logic. + +Possible values: + +- Positive integer. +- 0 — No limit. + +Default value: `0`. + ## background_pool_size {#background_pool_size} Sets the number of threads performing background merges and mutations for tables with MergeTree engines. This setting is also could be applied at server startup from the `default` profile configuration for backward compatibility at the ClickHouse server start. You can only increase the number of threads at runtime. To lower the number of threads you have to restart the server. By adjusting this setting, you manage CPU and disk load. Smaller pool size utilizes less CPU and disk resources, but background processes advance slower which might eventually impact query performance. @@ -1254,12 +1318,12 @@ Settings: ``` xml - /metrics - 8001 - true - true - true - + /metrics + 9363 + true + true + true + ``` ## query_log {#server_configuration_parameters-query-log} diff --git a/docs/en/operations/settings/settings-formats.md b/docs/en/operations/settings/settings-formats.md index 3580d83f704..172627c7c3e 100644 --- a/docs/en/operations/settings/settings-formats.md +++ b/docs/en/operations/settings/settings-formats.md @@ -1014,6 +1014,12 @@ Use Arrow FIXED_SIZE_BINARY type instead of Binary/String for FixedString column Enabled by default. +### output_format_arrow_compression_method {#output_format_arrow_compression_method} + +Compression method used in output Arrow format. Supported codecs: `lz4_frame`, `zstd`, `none` (uncompressed) + +Default value: `none`. + ## ORC format settings {#orc-format-settings} ### input_format_orc_import_nested {#input_format_orc_import_nested} @@ -1057,6 +1063,12 @@ Use ORC String type instead of Binary for String columns. Disabled by default. +### output_format_orc_compression_method {#output_format_orc_compression_method} + +Compression method used in output ORC format. Supported codecs: `lz4`, `snappy`, `zlib`, `zstd`, `none` (uncompressed) + +Default value: `none`. + ## Parquet format settings {#parquet-format-settings} ### input_format_parquet_import_nested {#input_format_parquet_import_nested} @@ -1112,6 +1124,12 @@ The version of Parquet format used in output format. Supported versions: `1.0`, Default value: `2.latest`. +### output_format_parquet_compression_method {#output_format_parquet_compression_method} + +Compression method used in output Parquet format. Supported codecs: `snappy`, `lz4`, `brotli`, `zstd`, `gzip`, `none` (uncompressed) + +Default value: `snappy`. + ## Hive format settings {#hive-format-settings} ### input_format_hive_text_fields_delimiter {#input_format_hive_text_fields_delimiter} @@ -1474,7 +1492,7 @@ Default value: `65505`. The name of table that will be used in the output INSERT statement. -Default value: `'table''`. +Default value: `table`. ### output_format_sql_insert_include_column_names {#output_format_sql_insert_include_column_names} @@ -1514,4 +1532,12 @@ Disabled by default. The maximum allowed size for String in RowBinary format. It prevents allocating large amount of memory in case of corrupted data. 0 means there is no limit. -Default value: `1GiB` +Default value: `1GiB`. + +## Native format settings {#native-format-settings} + +### input_format_native_allow_types_conversion {#input_format_native_allow_types_conversion} + +Allow types conversion in Native input format between columns from input data and requested columns. + +Enabled by default. diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index f960d2df98e..3c53f4fd0cf 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -966,10 +966,10 @@ This is an expert-level setting, and you shouldn't change it if you're just gett ## max_query_size {#settings-max_query_size} -The maximum part of a query that can be taken to RAM for parsing with the SQL parser. -The INSERT query also contains data for INSERT that is processed by a separate stream parser (that consumes O(1) RAM), which is not included in this restriction. +The maximum number of bytes of a query string parsed by the SQL parser. +Data in the VALUES clause of INSERT queries is processed by a separate stream parser (that consumes O(1) RAM) and not affected by this restriction. -Default value: 256 KiB. +Default value: 262144 (= 256 KiB). ## max_parser_depth {#max_parser_depth} @@ -1248,7 +1248,9 @@ Possible values: Default value: 1. :::warning -Disable this setting if you use [max_parallel_replicas](#settings-max_parallel_replicas). +Disable this setting if you use [max_parallel_replicas](#settings-max_parallel_replicas) without [parallel_replicas_custom_key](#settings-parallel_replicas_custom_key). +If [parallel_replicas_custom_key](#settings-parallel_replicas_custom_key) is set, disable this setting only if it's used on a cluster with multiple shards containing multiple replicas. +If it's used on a cluster with a single shard and multiple replicas, disabling this setting will have negative effects. ::: ## totals_mode {#totals-mode} @@ -1273,16 +1275,47 @@ Default value: `1`. **Additional Info** -This setting is useful for replicated tables with a sampling key. A query may be processed faster if it is executed on several servers in parallel. But the query performance may degrade in the following cases: +This options will produce different results depending on the settings used. + +:::warning +This setting will produce incorrect results when joins or subqueries are involved, and all tables don't meet certain requirements. See [Distributed Subqueries and max_parallel_replicas](../../sql-reference/operators/in.md/#max_parallel_replica-subqueries) for more details. +::: + +### Parallel processing using `SAMPLE` key + +A query may be processed faster if it is executed on several servers in parallel. But the query performance may degrade in the following cases: - The position of the sampling key in the partitioning key does not allow efficient range scans. - Adding a sampling key to the table makes filtering by other columns less efficient. - The sampling key is an expression that is expensive to calculate. - The cluster latency distribution has a long tail, so that querying more servers increases the query overall latency. -:::warning -This setting will produce incorrect results when joins or subqueries are involved, and all tables don't meet certain requirements. See [Distributed Subqueries and max_parallel_replicas](../../sql-reference/operators/in.md/#max_parallel_replica-subqueries) for more details. -::: +### Parallel processing using [parallel_replicas_custom_key](#settings-parallel_replicas_custom_key) + +This setting is useful for any replicated table. + +## parallel_replicas_custom_key {#settings-parallel_replicas_custom_key} + +An arbitrary integer expression that can be used to split work between replicas for a specific table. +The value can be any integer expression. +A query may be processed faster if it is executed on several servers in parallel but it depends on the used [parallel_replicas_custom_key](#settings-parallel_replicas_custom_key) +and [parallel_replicas_custom_key_filter_type](#settings-parallel_replicas_custom_key_filter_type). + +Simple expressions using primary keys are preferred. + +If the setting is used on a cluster that consists of a single shard with multiple replicas, those replicas will be converted into virtual shards. +Otherwise, it will behave same as for `SAMPLE` key, it will use multiple replicas of each shard. + +## parallel_replicas_custom_key_filter_type {#settings-parallel_replicas_custom_key_filter_type} + +How to use `parallel_replicas_custom_key` expression for splitting work between replicas. + +Possible values: + +- `default` — Use the default implementation using modulo operation on the `parallel_replicas_custom_key`. +- `range` — Split the entire value space of the expression in the ranges. This type of filtering is useful if values of `parallel_replicas_custom_key` are uniformly spread across the entire integer space, e.g. hash values. + +Default value: `default`. ## compile_expressions {#compile-expressions} @@ -1515,7 +1548,7 @@ Enables or disables asynchronous inserts. This makes sense only for insertion ov If enabled, the data is combined into batches before the insertion into tables, so it is possible to do small and frequent insertions into ClickHouse (up to 15000 queries per second) without buffer tables. -The data is inserted either after the [async_insert_max_data_size](#async-insert-max-data-size) is exceeded or after [async_insert_busy_timeout_ms](#async-insert-busy-timeout-ms) milliseconds since the first `INSERT` query. If the [async_insert_stale_timeout_ms](#async-insert-stale-timeout-ms) is set to a non-zero value, the data is inserted after `async_insert_stale_timeout_ms` milliseconds since the last query. +The data is inserted either after the [async_insert_max_data_size](#async-insert-max-data-size) is exceeded or after [async_insert_busy_timeout_ms](#async-insert-busy-timeout-ms) milliseconds since the first `INSERT` query. If the [async_insert_stale_timeout_ms](#async-insert-stale-timeout-ms) is set to a non-zero value, the data is inserted after `async_insert_stale_timeout_ms` milliseconds since the last query. Also the buffer will be flushed to disk if at least [async_insert_max_query_number](#async-insert-max-query-number) async insert queries per block were received. This last setting takes effect only if [async_insert_deduplicate](#async-insert-deduplicate) is enabled. If [wait_for_async_insert](#wait-for-async-insert) is enabled, every client will wait for the data to be processed and flushed to the table. Otherwise, the query would be processed almost instantly, even if the data is not inserted. diff --git a/docs/en/operations/storing-data.md b/docs/en/operations/storing-data.md index 203fe4e42d2..e019a3741cf 100644 --- a/docs/en/operations/storing-data.md +++ b/docs/en/operations/storing-data.md @@ -80,7 +80,7 @@ Required parameters: - `type` — `encrypted`. Otherwise the encrypted disk is not created. - `disk` — Type of disk for data storage. -- `key` — The key for encryption and decryption. Type: [Uint64](/docs/en/sql-reference/data-types/int-uint.md). You can use `key_hex` parameter to encrypt in hexadecimal form. +- `key` — The key for encryption and decryption. Type: [Uint64](/docs/en/sql-reference/data-types/int-uint.md). You can use `key_hex` parameter to encode the key in hexadecimal form. You can specify multiple keys using the `id` attribute (see example above). Optional parameters: @@ -135,11 +135,13 @@ Example of configuration for versions later or equal to 22.8: - -
- cache -
- + + +
+ cache +
+ + ``` @@ -159,11 +161,13 @@ Example of configuration for versions earlier than 22.8: - -
- s3 -
- + + +
+ s3 +
+ + ``` diff --git a/docs/en/operations/system-tables/opentelemetry_span_log.md b/docs/en/operations/system-tables/opentelemetry_span_log.md index 9d8aea46218..7d7d1ef1b04 100644 --- a/docs/en/operations/system-tables/opentelemetry_span_log.md +++ b/docs/en/operations/system-tables/opentelemetry_span_log.md @@ -15,6 +15,13 @@ Columns: - `operation_name` ([String](../../sql-reference/data-types/string.md)) — The name of the operation. +- `kind` ([Enum8](../../sql-reference/data-types/enum.md)) — The [SpanKind](https://opentelemetry.io/docs/reference/specification/trace/api/#spankind) of the span. + - `INTERNAL` — Indicates that the span represents an internal operation within an application. + - `SERVER` — Indicates that the span covers server-side handling of a synchronous RPC or other remote request. + - `CLIENT` — Indicates that the span describes a request to some remote service. + - `PRODUCER` — Indicates that the span describes the initiators of an asynchronous request. This parent span will often end before the corresponding child CONSUMER span, possibly even before the child span starts. + - `CONSUMER` - Indicates that the span describes a child of an asynchronous PRODUCER request. + - `start_time_us` ([UInt64](../../sql-reference/data-types/int-uint.md)) — The start time of the `trace span` (in microseconds). - `finish_time_us` ([UInt64](../../sql-reference/data-types/int-uint.md)) — The finish time of the `trace span` (in microseconds). @@ -42,6 +49,7 @@ trace_id: cdab0847-0d62-61d5-4d38-dd65b19a1914 span_id: 701487461015578150 parent_span_id: 2991972114672045096 operation_name: DB::Block DB::InterpreterSelectQuery::getSampleBlockImpl() +kind: INTERNAL start_time_us: 1612374594529090 finish_time_us: 1612374594529108 finish_date: 2021-02-03 diff --git a/docs/en/operations/utilities/clickhouse-format.md b/docs/en/operations/utilities/clickhouse-format.md index bf2e618b791..101310cc65e 100644 --- a/docs/en/operations/utilities/clickhouse-format.md +++ b/docs/en/operations/utilities/clickhouse-format.md @@ -27,7 +27,7 @@ $ clickhouse-format --query "select number from numbers(10) where number%2 order Result: -```text +```sql SELECT number FROM numbers(10) WHERE number % 2 @@ -54,7 +54,7 @@ $ clickhouse-format -n <<< "SELECT * FROM (SELECT 1 AS x UNION ALL SELECT 1 UNIO Result: -```text +```sql SELECT * FROM ( @@ -75,7 +75,7 @@ $ clickhouse-format --seed Hello --obfuscate <<< "SELECT cost_first_screen BETWE Result: -```text +```sql SELECT treasury_mammoth_hazelnut BETWEEN nutmeg AND span, CASE WHEN chive >= 116 THEN switching ELSE ANYTHING END; ``` @@ -87,7 +87,7 @@ $ clickhouse-format --seed World --obfuscate <<< "SELECT cost_first_screen BETWE Result: -```text +```sql SELECT horse_tape_summer BETWEEN folklore AND moccasins, CASE WHEN intestine >= 116 THEN nonconformist ELSE FORESTRY END; ``` @@ -99,7 +99,7 @@ $ clickhouse-format --backslash <<< "SELECT * FROM (SELECT 1 AS x UNION ALL SELE Result: -```text +```sql SELECT * \ FROM \ ( \ diff --git a/docs/en/operations/utilities/clickhouse-local.md b/docs/en/operations/utilities/clickhouse-local.md index a4fa5579638..08640b5c16b 100644 --- a/docs/en/operations/utilities/clickhouse-local.md +++ b/docs/en/operations/utilities/clickhouse-local.md @@ -6,29 +6,26 @@ sidebar_label: clickhouse-local # clickhouse-local -The `clickhouse-local` program enables you to perform fast processing on local files, without having to deploy and configure the ClickHouse server. +The `clickhouse-local` program enables you to perform fast processing on local files, without having to deploy and configure the ClickHouse server. It accepts data that represent tables and queries them using [ClickHouse SQL dialect](../../sql-reference/). `clickhouse-local` uses the same core as ClickHouse server, so it supports most of the features and the same set of formats and table engines. -Accepts data that represent tables and queries them using [ClickHouse SQL dialect](../../sql-reference/). - -`clickhouse-local` uses the same core as ClickHouse server, so it supports most of the features and the same set of formats and table engines. - -By default `clickhouse-local` does not have access to data on the same host, but it supports loading server configuration using `--config-file` argument. - -:::warning -It is not recommended to load production server configuration into `clickhouse-local` because data can be damaged in case of human error. -::: - -For temporary data, a unique temporary data directory is created by default. +By default `clickhouse-local` has access to data on the same host, and it does not depend on the server's configuration. It also supports loading server configuration using `--config-file` argument. For temporary data, a unique temporary data directory is created by default. ## Usage {#usage} -Basic usage: +Basic usage (Linux): ``` bash -$ clickhouse-local --structure "table_structure" --input-format "format_of_incoming_data" \ - --query "query" +$ clickhouse-local --structure "table_structure" --input-format "format_of_incoming_data" --query "query" ``` +Basic usage (Mac): + +``` bash +$ ./clickhouse local --structure "table_structure" --input-format "format_of_incoming_data" --query "query" +``` + +Also supported on Windows through WSL2. + Arguments: - `-S`, `--structure` — table structure for input data. diff --git a/docs/en/sql-reference/aggregate-functions/reference/exponentialmovingaverage.md b/docs/en/sql-reference/aggregate-functions/reference/exponentialmovingaverage.md index 2587bc5533f..5546ade1758 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/exponentialmovingaverage.md +++ b/docs/en/sql-reference/aggregate-functions/reference/exponentialmovingaverage.md @@ -11,15 +11,15 @@ sidebar_title: exponentialMovingAverage **Syntax** ```sql -exponentialMovingAverage(x)(value, timestamp) +exponentialMovingAverage(x)(value, timeunit) ``` -Each `value` corresponds to the determinate `timestamp`. The half-life `x` is the time lag at which the exponential weights decay by one-half. The function returns a weighted average: the older the time point, the less weight the corresponding value is considered to be. +Each `value` corresponds to the determinate `timeunit`. The half-life `x` is the time lag at which the exponential weights decay by one-half. The function returns a weighted average: the older the time point, the less weight the corresponding value is considered to be. **Arguments** - `value` — Value. [Integer](../../../sql-reference/data-types/int-uint.md), [Float](../../../sql-reference/data-types/float.md) or [Decimal](../../../sql-reference/data-types/decimal.md). -- `timestamp` — Timestamp. [Integer](../../../sql-reference/data-types/int-uint.md), [Float](../../../sql-reference/data-types/float.md) or [Decimal](../../../sql-reference/data-types/decimal.md). +- `timeunit` — Timeunit. [Integer](../../../sql-reference/data-types/int-uint.md), [Float](../../../sql-reference/data-types/float.md) or [Decimal](../../../sql-reference/data-types/decimal.md). Timeunit is not timestamp (seconds), it's -- an index of the time interval. Can be calculated using [intDiv](../../functions/arithmetic-functions/#intdiva-b). **Parameters** @@ -148,3 +148,58 @@ Result: │ 1 │ 49 │ 0.825 │ █████████████████████████████████████████▎│ └───────┴──────┴──────────────────────┴────────────────────────────────────────────┘ ``` + +```sql +CREATE TABLE data +ENGINE = Memory AS +SELECT + 10 AS value, + toDateTime('2020-01-01') + (3600 * number) AS time +FROM numbers_mt(10); + + +-- Calculate timeunit using intDiv +SELECT + value, + time, + exponentialMovingAverage(1)(value, intDiv(toUInt32(time), 3600)) OVER (ORDER BY time ASC) AS res, + intDiv(toUInt32(time), 3600) AS timeunit +FROM data +ORDER BY time ASC; + +┌─value─┬────────────────time─┬─────────res─┬─timeunit─┐ +│ 10 │ 2020-01-01 00:00:00 │ 5 │ 438288 │ +│ 10 │ 2020-01-01 01:00:00 │ 7.5 │ 438289 │ +│ 10 │ 2020-01-01 02:00:00 │ 8.75 │ 438290 │ +│ 10 │ 2020-01-01 03:00:00 │ 9.375 │ 438291 │ +│ 10 │ 2020-01-01 04:00:00 │ 9.6875 │ 438292 │ +│ 10 │ 2020-01-01 05:00:00 │ 9.84375 │ 438293 │ +│ 10 │ 2020-01-01 06:00:00 │ 9.921875 │ 438294 │ +│ 10 │ 2020-01-01 07:00:00 │ 9.9609375 │ 438295 │ +│ 10 │ 2020-01-01 08:00:00 │ 9.98046875 │ 438296 │ +│ 10 │ 2020-01-01 09:00:00 │ 9.990234375 │ 438297 │ +└───────┴─────────────────────┴─────────────┴──────────┘ + + +-- Calculate timeunit using toRelativeHourNum +SELECT + value, + time, + exponentialMovingAverage(1)(value, toRelativeHourNum(time)) OVER (ORDER BY time ASC) AS res, + toRelativeHourNum(time) AS timeunit +FROM data +ORDER BY time ASC; + +┌─value─┬────────────────time─┬─────────res─┬─timeunit─┐ +│ 10 │ 2020-01-01 00:00:00 │ 5 │ 438288 │ +│ 10 │ 2020-01-01 01:00:00 │ 7.5 │ 438289 │ +│ 10 │ 2020-01-01 02:00:00 │ 8.75 │ 438290 │ +│ 10 │ 2020-01-01 03:00:00 │ 9.375 │ 438291 │ +│ 10 │ 2020-01-01 04:00:00 │ 9.6875 │ 438292 │ +│ 10 │ 2020-01-01 05:00:00 │ 9.84375 │ 438293 │ +│ 10 │ 2020-01-01 06:00:00 │ 9.921875 │ 438294 │ +│ 10 │ 2020-01-01 07:00:00 │ 9.9609375 │ 438295 │ +│ 10 │ 2020-01-01 08:00:00 │ 9.98046875 │ 438296 │ +│ 10 │ 2020-01-01 09:00:00 │ 9.990234375 │ 438297 │ +└───────┴─────────────────────┴─────────────┴──────────┘ +``` diff --git a/docs/en/sql-reference/functions/date-time-functions.md b/docs/en/sql-reference/functions/date-time-functions.md index 49b0bc25edd..ef0475027dd 100644 --- a/docs/en/sql-reference/functions/date-time-functions.md +++ b/docs/en/sql-reference/functions/date-time-functions.md @@ -1126,15 +1126,48 @@ Rounds the time to the half hour. ## toYYYYMM -Converts a date or date with time to a UInt32 number containing the year and month number (YYYY \* 100 + MM). +Converts a date or date with time to a UInt32 number containing the year and month number (YYYY \* 100 + MM). Accepts a second optional timezone argument. If provided, the timezone must be a string constant. + +### example +```sql +SELECT + toYYYYMM(now(), 'US/Eastern') +``` +```response +┌─toYYYYMM(now(), 'US/Eastern')─┐ +│ 202303 │ +└───────────────────────────────┘ +``` ## toYYYYMMDD -Converts a date or date with time to a UInt32 number containing the year and month number (YYYY \* 10000 + MM \* 100 + DD). +Converts a date or date with time to a UInt32 number containing the year and month number (YYYY \* 10000 + MM \* 100 + DD). Accepts a second optional timezone argument. If provided, the timezone must be a string constant. + +### example +```sql +SELECT + toYYYYMMDD(now(), 'US/Eastern') +``` +```response +┌─toYYYYMMDD(now(), 'US/Eastern')─┐ +│ 20230302 │ +└─────────────────────────────────┘ +``` ## toYYYYMMDDhhmmss -Converts a date or date with time to a UInt64 number containing the year and month number (YYYY \* 10000000000 + MM \* 100000000 + DD \* 1000000 + hh \* 10000 + mm \* 100 + ss). +Converts a date or date with time to a UInt64 number containing the year and month number (YYYY \* 10000000000 + MM \* 100000000 + DD \* 1000000 + hh \* 10000 + mm \* 100 + ss). Accepts a second optional timezone argument. If provided, the timezone must be a string constant. + +### example +```sql +SELECT + toYYYYMMDDhhmmss(now(), 'US/Eastern') +``` +```response +┌─toYYYYMMDDhhmmss(now(), 'US/Eastern')─┐ +│ 20230302112209 │ +└───────────────────────────────────────┘ +``` ## addYears, addMonths, addWeeks, addDays, addHours, addMinutes, addSeconds, addQuarters diff --git a/docs/en/sql-reference/functions/ip-address-functions.md b/docs/en/sql-reference/functions/ip-address-functions.md index 3843ca0fc36..47058a28d12 100644 --- a/docs/en/sql-reference/functions/ip-address-functions.md +++ b/docs/en/sql-reference/functions/ip-address-functions.md @@ -280,12 +280,20 @@ SELECT ## toIPv4OrDefault(string) -Same as `toIPv4`, but if the IPv4 address has an invalid format, it returns 0. +Same as `toIPv4`, but if the IPv4 address has an invalid format, it returns `0.0.0.0` (0 IPv4). ## toIPv4OrNull(string) Same as `toIPv4`, but if the IPv4 address has an invalid format, it returns null. +## toIPv6OrDefault(string) + +Same as `toIPv6`, but if the IPv6 address has an invalid format, it returns `::` (0 IPv6). + +## toIPv6OrNull(string) + +Same as `toIPv6`, but if the IPv6 address has an invalid format, it returns null. + ## toIPv6 Converts a string form of IPv6 address to [IPv6](../../sql-reference/data-types/domains/ipv6.md) type. If the IPv6 address has an invalid format, returns an empty value. diff --git a/docs/en/sql-reference/functions/string-functions.md b/docs/en/sql-reference/functions/string-functions.md index 845be6e04c7..f3c5b20f886 100644 --- a/docs/en/sql-reference/functions/string-functions.md +++ b/docs/en/sql-reference/functions/string-functions.md @@ -330,7 +330,7 @@ repeat(s, n) **Arguments** - `s` — The string to repeat. [String](../../sql-reference/data-types/string.md). -- `n` — The number of times to repeat the string. [UInt](../../sql-reference/data-types/int-uint.md). +- `n` — The number of times to repeat the string. [UInt or Int](../../sql-reference/data-types/int-uint.md). **Returned value** diff --git a/docs/en/sql-reference/operators/in.md b/docs/en/sql-reference/operators/in.md index 58119cfc4f5..0599a50c0a4 100644 --- a/docs/en/sql-reference/operators/in.md +++ b/docs/en/sql-reference/operators/in.md @@ -233,8 +233,9 @@ If `some_predicate` is not selective enough, it will return large amount of data ### Distributed Subqueries and max_parallel_replicas -When max_parallel_replicas is greater than 1, distributed queries are further transformed. For example, the following: +When [max_parallel_replicas](#settings-max_parallel_replicas) is greater than 1, distributed queries are further transformed. +For example, the following: ```sql SELECT CounterID, count() FROM distributed_table_1 WHERE UserID IN (SELECT UserID FROM local_table_2 WHERE CounterID < 100) SETTINGS max_parallel_replicas=3 @@ -247,8 +248,12 @@ SELECT CounterID, count() FROM local_table_1 WHERE UserID IN (SELECT UserID FROM SETTINGS parallel_replicas_count=3, parallel_replicas_offset=M ``` -where M is between 1 and 3 depending on which replica the local query is executing on. These settings affect every MergeTree-family table in the query and have the same effect as applying `SAMPLE 1/3 OFFSET (M-1)/3` on each table. +where M is between 1 and 3 depending on which replica the local query is executing on. -Therefore adding the max_parallel_replicas setting will only produce correct results if both tables have the same replication scheme and are sampled by UserID or a subkey of it. In particular, if local_table_2 does not have a sampling key, incorrect results will be produced. The same rule applies to JOIN. +These settings affect every MergeTree-family table in the query and have the same effect as applying `SAMPLE 1/3 OFFSET (M-1)/3` on each table. + +Therefore adding the [max_parallel_replicas](#settings-max_parallel_replicas) setting will only produce correct results if both tables have the same replication scheme and are sampled by UserID or a subkey of it. In particular, if local_table_2 does not have a sampling key, incorrect results will be produced. The same rule applies to JOIN. One workaround if local_table_2 does not meet the requirements, is to use `GLOBAL IN` or `GLOBAL JOIN`. + +If a table doesn't have a sampling key, more flexible options for [parallel_replicas_custom_key](#settings-parallel_replicas_custom_key) can be used that can produce different and more optimal behaviour. diff --git a/docs/en/sql-reference/statements/alter/comment.md b/docs/en/sql-reference/statements/alter/comment.md index f8742765619..cc49c6abf80 100644 --- a/docs/en/sql-reference/statements/alter/comment.md +++ b/docs/en/sql-reference/statements/alter/comment.md @@ -16,7 +16,7 @@ ALTER TABLE [db].name [ON CLUSTER cluster] MODIFY COMMENT 'Comment' **Examples** -Creating a table with comment (for more information, see the [COMMENT] clause(../../../sql-reference/statements/create/table.md#comment-table)): +Creating a table with comment (for more information, see the [COMMENT](../../../sql-reference/statements/create/table.md#comment-table) clause): ``` sql CREATE TABLE table_with_comment diff --git a/docs/en/sql-reference/statements/create/table.md b/docs/en/sql-reference/statements/create/table.md index 54977e1b0ab..de39d960476 100644 --- a/docs/en/sql-reference/statements/create/table.md +++ b/docs/en/sql-reference/statements/create/table.md @@ -110,25 +110,23 @@ If the type is not `Nullable` and if `NULL` is specified, it will be treated as See also [data_type_default_nullable](../../../operations/settings/settings.md#data_type_default_nullable) setting. -## Default Values +## Default Values {#default_values} -The column description can specify an expression for a default value, in one of the following ways: `DEFAULT expr`, `MATERIALIZED expr`, `ALIAS expr`. +The column description can specify a default value expression in the form of `DEFAULT expr`, `MATERIALIZED expr`, or `ALIAS expr`. Example: `URLDomain String DEFAULT domain(URL)`. -Example: `URLDomain String DEFAULT domain(URL)`. +The expression `expr` is optional. If it is omitted, the column type must be specified explicitly and the default value will be `0` for numeric columns, `''` (the empty string) for string columns, `[]` (the empty array) for array columns, `1970-01-01` for date columns, or `NULL` for nullable columns. -If an expression for the default value is not defined, the default values will be set to zeros for numbers, empty strings for strings, empty arrays for arrays, and `1970-01-01` for dates or zero unix timestamp for DateTime, NULL for Nullable. +The column type of a default value column can be omitted in which case it is infered from `expr`'s type. For example the type of column `EventDate DEFAULT toDate(EventTime)` will be date. -If the default expression is defined, the column type is optional. If there isn’t an explicitly defined type, the default expression type is used. Example: `EventDate DEFAULT toDate(EventTime)` – the ‘Date’ type will be used for the ‘EventDate’ column. +If both a data type and a default value expression are specified, an implicit type casting function inserted which converts the expression to the specified type. Example: `Hits UInt32 DEFAULT 0` is internally represented as `Hits UInt32 DEFAULT toUInt32(0)`. -If the data type and default expression are defined explicitly, this expression will be cast to the specified type using type casting functions. Example: `Hits UInt32 DEFAULT 0` means the same thing as `Hits UInt32 DEFAULT toUInt32(0)`. - -Default expressions may be defined as an arbitrary expression from table constants and columns. When creating and changing the table structure, it checks that expressions do not contain loops. For INSERT, it checks that expressions are resolvable – that all columns they can be calculated from have been passed. +A default value expression `expr` may reference arbitrary table columns and constants. ClickHouse checks that changes of the table structure do not introduce loops in the expression calculation. For INSERT, it checks that expressions are resolvable – that all columns they can be calculated from have been passed. ### DEFAULT `DEFAULT expr` -Normal default value. If the INSERT query does not specify the corresponding column, it will be filled in by computing the corresponding expression. +Normal default value. If the value of such a column is not specified in an INSERT query, it is computed from `expr`. Example: @@ -154,9 +152,9 @@ SELECT * FROM test; `MATERIALIZED expr` -Materialized expression. Such a column can’t be specified for INSERT, because it is always calculated. -For an INSERT without a list of columns, these columns are not considered. -In addition, this column is not substituted when using an asterisk in a SELECT query. This is to preserve the invariant that the dump obtained using `SELECT *` can be inserted back into the table using INSERT without specifying the list of columns. +Materialized expression. Values of such columns are always calculated, they cannot be specified in INSERT queries. + +Also, default value columns of this type are not included in the result of `SELECT *`. This is to preserve the invariant that the result of a `SELECT *` can always be inserted back into the table using `INSERT`. This behavior can be disabled with setting `asterisk_include_materialized_columns`. Example: @@ -192,8 +190,9 @@ SELECT * FROM test SETTINGS asterisk_include_materialized_columns=1; `EPHEMERAL [expr]` -Ephemeral column. Such a column isn't stored in the table and cannot be SELECTed, but can be referenced in the defaults of CREATE statement. If `expr` is omitted type for column is required. -INSERT without list of columns will skip such column, so SELECT/INSERT invariant is preserved - the dump obtained using `SELECT *` can be inserted back into the table using INSERT without specifying the list of columns. +Ephemeral column. Columns of this type are not stored in the table and it is not possible to SELECT from them. The only purpose of ephemeral columns is to build default value expressions of other columns from them. + +An insert without explicitly specified columns will skip columns of this type. This is to preserve the invariant that the result of a `SELECT *` can always be inserted back into the table using `INSERT`. Example: @@ -205,7 +204,7 @@ CREATE OR REPLACE TABLE test hexed FixedString(4) DEFAULT unhex(unhexed) ) ENGINE = MergeTree -ORDER BY id +ORDER BY id; INSERT INTO test (id, unhexed) Values (1, '5a90b714'); @@ -227,9 +226,9 @@ hex(hexed): 5A90B714 `ALIAS expr` -Synonym. Such a column isn’t stored in the table at all. -Its values can’t be inserted in a table, and it is not substituted when using an asterisk in a SELECT query. -It can be used in SELECTs if the alias is expanded during query parsing. +Calculated columns (synonym). Column of this type are not stored in the table and it is not possible to INSERT values into them. + +When SELECT queries explicitly reference columns of this type, the value is computed at query time from `expr`. By default, `SELECT *` excludes ALIAS columns. This behavior can be disabled with setting `asteriks_include_alias_columns`. When using the ALTER query to add new columns, old data for these columns is not written. Instead, when reading old data that does not have values for the new columns, expressions are computed on the fly by default. However, if running the expressions requires different columns that are not indicated in the query, these columns will additionally be read, but only for the blocks of data that need it. @@ -394,15 +393,15 @@ These codecs are designed to make compression more effective by using specific f #### DoubleDelta -`DoubleDelta` — Calculates delta of deltas and writes it in compact binary form. Optimal compression rates are achieved for monotonic sequences with a constant stride, such as time series data. Can be used with any fixed-width type. Implements the algorithm used in Gorilla TSDB, extending it to support 64-bit types. Uses 1 extra bit for 32-byte deltas: 5-bit prefixes instead of 4-bit prefixes. For additional information, see Compressing Time Stamps in [Gorilla: A Fast, Scalable, In-Memory Time Series Database](http://www.vldb.org/pvldb/vol8/p1816-teller.pdf). +`DoubleDelta(bytes_size)` — Calculates delta of deltas and writes it in compact binary form. Possible `bytes_size` values: 1, 2, 4, 8, the default value is `sizeof(type)` if equal to 1, 2, 4, or 8. In all other cases, it’s 1. Optimal compression rates are achieved for monotonic sequences with a constant stride, such as time series data. Can be used with any fixed-width type. Implements the algorithm used in Gorilla TSDB, extending it to support 64-bit types. Uses 1 extra bit for 32-bit deltas: 5-bit prefixes instead of 4-bit prefixes. For additional information, see Compressing Time Stamps in [Gorilla: A Fast, Scalable, In-Memory Time Series Database](http://www.vldb.org/pvldb/vol8/p1816-teller.pdf). #### Gorilla -`Gorilla` — Calculates XOR between current and previous floating point value and writes it in compact binary form. The smaller the difference between consecutive values is, i.e. the slower the values of the series changes, the better the compression rate. Implements the algorithm used in Gorilla TSDB, extending it to support 64-bit types. For additional information, see section 4.1 in [Gorilla: A Fast, Scalable, In-Memory Time Series Database](https://doi.org/10.14778/2824032.2824078). +`Gorilla(bytes_size)` — Calculates XOR between current and previous floating point value and writes it in compact binary form. The smaller the difference between consecutive values is, i.e. the slower the values of the series changes, the better the compression rate. Implements the algorithm used in Gorilla TSDB, extending it to support 64-bit types. Possible `bytes_size` values: 1, 2, 4, 8, the default value is `sizeof(type)` if equal to 1, 2, 4, or 8. In all other cases, it’s 1. For additional information, see section 4.1 in [Gorilla: A Fast, Scalable, In-Memory Time Series Database](https://doi.org/10.14778/2824032.2824078). #### FPC -`FPC` - Repeatedly predicts the next floating point value in the sequence using the better of two predictors, then XORs the actual with the predicted value, and leading-zero compresses the result. Similar to Gorilla, this is efficient when storing a series of floating point values that change slowly. For 64-bit values (double), FPC is faster than Gorilla, for 32-bit values your mileage may vary. For a detailed description of the algorithm see [High Throughput Compression of Double-Precision Floating-Point Data](https://userweb.cs.txstate.edu/~burtscher/papers/dcc07a.pdf). +`FPC(level, float_size)` - Repeatedly predicts the next floating point value in the sequence using the better of two predictors, then XORs the actual with the predicted value, and leading-zero compresses the result. Similar to Gorilla, this is efficient when storing a series of floating point values that change slowly. For 64-bit values (double), FPC is faster than Gorilla, for 32-bit values your mileage may vary. Possible `level` values: 1-28, the default value is 12. Possible `float_size` values: 4, 8, the default value is `sizeof(type)` if type is Float. In all other cases, it’s 4. For a detailed description of the algorithm see [High Throughput Compression of Double-Precision Floating-Point Data](https://userweb.cs.txstate.edu/~burtscher/papers/dcc07a.pdf). #### T64 @@ -474,7 +473,7 @@ ENGINE = MergeTree ORDER BY x; ClickHouse supports temporary tables which have the following characteristics: - Temporary tables disappear when the session ends, including if the connection is lost. -- A temporary table uses the Memory engine only. +- A temporary table uses the Memory table engine when engine is not specified and it may use any table engine except Replicated and `KeeperMap` engines. - The DB can’t be specified for a temporary table. It is created outside of databases. - Impossible to create a temporary table with distributed DDL query on all cluster servers (by using `ON CLUSTER`): this table exists only in the current session. - If a temporary table has the same name as another one and a query specifies the table name without specifying the DB, the temporary table will be used. @@ -488,7 +487,7 @@ CREATE TEMPORARY TABLE [IF NOT EXISTS] table_name name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1], name2 [type2] [DEFAULT|MATERIALIZED|ALIAS expr2], ... -) +) [ENGINE = engine] ``` In most cases, temporary tables are not created manually, but when using external data for a query, or for distributed `(GLOBAL) IN`. For more information, see the appropriate sections @@ -576,7 +575,7 @@ SELECT * FROM base.t1; You can add a comment to the table when you creating it. :::note -The comment is supported for all table engines except [Kafka](../../../engines/table-engines/integrations/kafka.md), [RabbitMQ](../../../engines/table-engines/integrations/rabbitmq.md) and [EmbeddedRocksDB](../../../engines/table-engines/integrations/embedded-rocksdb.md). +The comment clause is supported by all table engines except [Kafka](../../../engines/table-engines/integrations/kafka.md), [RabbitMQ](../../../engines/table-engines/integrations/rabbitmq.md) and [EmbeddedRocksDB](../../../engines/table-engines/integrations/embedded-rocksdb.md). ::: diff --git a/docs/en/sql-reference/statements/create/view.md b/docs/en/sql-reference/statements/create/view.md index acdede3c673..0def42259ab 100644 --- a/docs/en/sql-reference/statements/create/view.md +++ b/docs/en/sql-reference/statements/create/view.md @@ -70,6 +70,12 @@ A materialized view is implemented as follows: when inserting data to the table Materialized views in ClickHouse use **column names** instead of column order during insertion into destination table. If some column names are not present in the `SELECT` query result, ClickHouse uses a default value, even if the column is not [Nullable](../../data-types/nullable.md). A safe practice would be to add aliases for every column when using Materialized views. Materialized views in ClickHouse are implemented more like insert triggers. If there’s some aggregation in the view query, it’s applied only to the batch of freshly inserted data. Any changes to existing data of source table (like update, delete, drop partition, etc.) does not change the materialized view. + +Materialized views in ClickHouse do not have deterministic behaviour in case of errors. This means that blocks that had been already written will be preserved in the destination table, but all blocks after error will not. + +By default if pushing to one of views fails, then the INSERT query will fail too, and some blocks may not be written to the destination table. This can be changed using `materialized_views_ignore_errors` setting (you should set it for `INSERT` query), if you will set `materialized_views_ignore_errors=true`, then any errors while pushing to views will be ignored and all blocks will be written to the destination table. + +Also note, that `materialized_views_ignore_errors` set to `true` by default for `system.*_log` tables. ::: If you specify `POPULATE`, the existing table data is inserted into the view when creating it, as if making a `CREATE TABLE ... AS SELECT ...` . Otherwise, the query contains only the data inserted in the table after creating the view. We **do not recommend** using `POPULATE`, since data inserted in the table during the view creation will not be inserted in it. diff --git a/docs/en/sql-reference/statements/grant.md b/docs/en/sql-reference/statements/grant.md index 3383ea70a2b..1d9b2c9ea30 100644 --- a/docs/en/sql-reference/statements/grant.md +++ b/docs/en/sql-reference/statements/grant.md @@ -105,7 +105,8 @@ Hierarchy of privileges: - [CREATE](#grant-create) - `CREATE DATABASE` - `CREATE TABLE` - - `CREATE TEMPORARY TABLE` + - `CREATE ARBITRARY TEMPORARY TABLE` + - `CREATE TEMPORARY TABLE` - `CREATE VIEW` - `CREATE DICTIONARY` - `CREATE FUNCTION` @@ -313,7 +314,8 @@ Allows executing [CREATE](../../sql-reference/statements/create/index.md) and [A - `CREATE`. Level: `GROUP` - `CREATE DATABASE`. Level: `DATABASE` - `CREATE TABLE`. Level: `TABLE` - - `CREATE TEMPORARY TABLE`. Level: `GLOBAL` + - `CREATE ARBITRARY TEMPORARY TABLE`. Level: `GLOBAL` + - `CREATE TEMPORARY TABLE`. Level: `GLOBAL` - `CREATE VIEW`. Level: `VIEW` - `CREATE DICTIONARY`. Level: `DICTIONARY` diff --git a/docs/en/sql-reference/statements/insert-into.md b/docs/en/sql-reference/statements/insert-into.md index 03a4ab3453c..f2d590d196b 100644 --- a/docs/en/sql-reference/statements/insert-into.md +++ b/docs/en/sql-reference/statements/insert-into.md @@ -91,6 +91,13 @@ INSERT INTO t FORMAT TabSeparated You can insert data separately from the query by using the command-line client or the HTTP interface. For more information, see the section “[Interfaces](../../interfaces)”. +:::note +If you want to specify `SETTINGS` for `INSERT` query then you have to do it _before_ `FORMAT` clause since everything after `FORMAT format_name` is treated as data. For example: +```sql +INSERT INTO table SETTINGS ... FORMAT format_name data_set +``` +::: + ## Constraints If table has [constraints](../../sql-reference/statements/create/table.md#constraints), their expressions will be checked for each row of inserted data. If any of those constraints is not satisfied — server will raise an exception containing constraint name and expression, the query will be stopped. diff --git a/docs/en/sql-reference/table-functions/file.md b/docs/en/sql-reference/table-functions/file.md index d2ef66dde73..594c328c3ff 100644 --- a/docs/en/sql-reference/table-functions/file.md +++ b/docs/en/sql-reference/table-functions/file.md @@ -6,21 +6,22 @@ sidebar_label: file # file -Creates a table from a file. This table function is similar to [url](../../sql-reference/table-functions/url.md) and [hdfs](../../sql-reference/table-functions/hdfs.md) ones. +Creates a table from a file. This table function is similar to [url](/docs/en/sql-reference/table-functions/url.md) and [hdfs](/docs/en/sql-reference/table-functions/hdfs.md) ones. -`file` function can be used in `SELECT` and `INSERT` queries on data in [File](../../engines/table-engines/special/file.md) tables. +`file` function can be used in `SELECT` and `INSERT` queries on data in [File](/docs/en/engines/table-engines/special/file.md) tables. **Syntax** ``` sql -file(path [,format] [,structure]) +file(path [,format] [,structure] [,compression]) ``` **Parameters** -- `path` — The relative path to the file from [user_files_path](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-user_files_path). Path to file support following globs in read-only mode: `*`, `?`, `{abc,def}` and `{N..M}` where `N`, `M` — numbers, `'abc', 'def'` — strings. -- `format` — The [format](../../interfaces/formats.md#formats) of the file. +- `path` — The relative path to the file from [user_files_path](/docs/en/operations/server-configuration-parameters/settings.md#server_configuration_parameters-user_files_path). Path to file support following globs in read-only mode: `*`, `?`, `{abc,def}` and `{N..M}` where `N`, `M` — numbers, `'abc', 'def'` — strings. +- `format` — The [format](/docs/en/interfaces/formats.md#formats) of the file. - `structure` — Structure of the table. Format: `'column1_name column1_type, column2_name column2_type, ...'`. +- `compression` — The existing compression type when used in a `SELECT` query, or the desired compression type when used in an `INSERT` query. The supported compression types are `gz`, `br`, `xz`, `zst`, `lz4`, and `bz2`. **Returned value** @@ -53,7 +54,7 @@ SELECT * FROM file('test.csv', 'CSV', 'column1 UInt32, column2 UInt32, column3 U └─────────┴─────────┴─────────┘ ``` -Getting the first 10 lines of a table that contains 3 columns of [UInt32](../../sql-reference/data-types/int-uint.md) type from a CSV file: +Getting the first 10 lines of a table that contains 3 columns of [UInt32](/docs/en/sql-reference/data-types/int-uint.md) type from a CSV file: ``` sql SELECT * FROM file('test.csv', 'CSV', 'column1 UInt32, column2 UInt32, column3 UInt32') LIMIT 10; @@ -143,4 +144,4 @@ SELECT count(*) FROM file('big_dir/**/file002', 'CSV', 'name String, value UInt3 **See Also** -- [Virtual columns](../../engines/table-engines/index.md#table_engines-virtual_columns) +- [Virtual columns](/docs/en/engines/table-engines/index.md#table_engines-virtual_columns) diff --git a/docs/ru/engines/table-engines/mergetree-family/collapsingmergetree.md b/docs/ru/engines/table-engines/mergetree-family/collapsingmergetree.md index dac490468d0..e3b4238a200 100644 --- a/docs/ru/engines/table-engines/mergetree-family/collapsingmergetree.md +++ b/docs/ru/engines/table-engines/mergetree-family/collapsingmergetree.md @@ -89,7 +89,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] └─────────────────────┴───────────┴──────────┴──────┘ ``` -Первая строка отменяет предыдущее состояние объекта (пользователя). Она должен повторять все поля из ключа сортировки для отменённого состояния за исключением `Sign`. +Первая строка отменяет предыдущее состояние объекта (пользователя). Она должна повторять все поля из ключа сортировки для отменённого состояния за исключением `Sign`. Вторая строка содержит текущее состояние. diff --git a/docs/ru/engines/table-engines/mergetree-family/mergetree.md b/docs/ru/engines/table-engines/mergetree-family/mergetree.md index 7269cc023e4..24e0f8dbbb8 100644 --- a/docs/ru/engines/table-engines/mergetree-family/mergetree.md +++ b/docs/ru/engines/table-engines/mergetree-family/mergetree.md @@ -584,7 +584,7 @@ TTL d + INTERVAL 1 MONTH GROUP BY k1, k2 SET x = max(x), y = min(y); Данные с истекшим `TTL` удаляются, когда ClickHouse мёржит куски данных. -Когда ClickHouse видит, что некоторые данные устарели, он выполняет внеплановые мёржи. Для управление частотой подобных мёржей, можно задать настройку `merge_with_ttl_timeout`. Если её значение слишком низкое, придется выполнять много внеплановых мёржей, которые могут начать потреблять значительную долю ресурсов сервера. +Когда ClickHouse видит, что некоторые данные устарели, он выполняет внеплановые мёржи. Для управления частотой подобных мёржей, можно задать настройку `merge_with_ttl_timeout`. Если её значение слишком низкое, придется выполнять много внеплановых мёржей, которые могут начать потреблять значительную долю ресурсов сервера. Если вы выполните запрос `SELECT` между слияниями вы можете получить устаревшие данные. Чтобы избежать этого используйте запрос [OPTIMIZE](../../../engines/table-engines/mergetree-family/mergetree.md#misc_operations-optimize) перед `SELECT`. @@ -679,7 +679,7 @@ TTL d + INTERVAL 1 MONTH GROUP BY k1, k2 SET x = max(x), y = min(y); - `policy_name_N` — название политики. Названия политик должны быть уникальны. - `volume_name_N` — название тома. Названия томов должны быть уникальны. - `disk` — диск, находящийся внутри тома. -- `max_data_part_size_bytes` — максимальный размер куска данных, который может находится на любом из дисков этого тома. Если в результате слияния размер куска ожидается больше, чем max_data_part_size_bytes, то этот кусок будет записан в следующий том. В основном эта функция позволяет хранить новые / мелкие куски на горячем (SSD) томе и перемещать их на холодный (HDD) том, когда они достигают большого размера. Не используйте этот параметр, если политика имеет только один том. +- `max_data_part_size_bytes` — максимальный размер куска данных, который может находиться на любом из дисков этого тома. Если в результате слияния размер куска ожидается больше, чем max_data_part_size_bytes, то этот кусок будет записан в следующий том. В основном эта функция позволяет хранить новые / мелкие куски на горячем (SSD) томе и перемещать их на холодный (HDD) том, когда они достигают большого размера. Не используйте этот параметр, если политика имеет только один том. - `move_factor` — доля доступного свободного места на томе, если места становится меньше, то данные начнут перемещение на следующий том, если он есть (по умолчанию 0.1). Для перемещения куски сортируются по размеру от большего к меньшему (по убыванию) и выбираются куски, совокупный размер которых достаточен для соблюдения условия `move_factor`, если совокупный размер всех партов недостаточен, будут перемещены все парты. - `prefer_not_to_merge` — Отключает слияние кусков данных, хранящихся на данном томе. Если данная настройка включена, то слияние данных, хранящихся на данном томе, не допускается. Это позволяет контролировать работу ClickHouse с медленными дисками. @@ -730,7 +730,7 @@ TTL d + INTERVAL 1 MONTH GROUP BY k1, k2 SET x = max(x), y = min(y); В приведенном примере, политика `hdd_in_order` реализует прицип [round-robin](https://ru.wikipedia.org/wiki/Round-robin_(%D0%B0%D0%BB%D0%B3%D0%BE%D1%80%D0%B8%D1%82%D0%BC)). Так как в политике есть всего один том (`single`), то все записи производятся на его диски по круговому циклу. Такая политика может быть полезна при наличии в системе нескольких похожих дисков, но при этом не сконфигурирован RAID. Учтите, что каждый отдельный диск ненадёжен и чтобы не потерять важные данные это необходимо скомпенсировать за счет хранения данных в трёх копиях. -Если система содержит диски различных типов, то может пригодиться политика `moving_from_ssd_to_hdd`. В томе `hot` находится один SSD-диск (`fast_ssd`), а также задается ограничение на максимальный размер куска, который может храниться на этом томе (1GB). Все куски такой таблицы больше 1GB будут записываться сразу на том `cold`, в котором содержится один HDD-диск `disk1`. Также, при заполнении диска `fast_ssd` более чем на 80% данные будут переносится на диск `disk1` фоновым процессом. +Если система содержит диски различных типов, то может пригодиться политика `moving_from_ssd_to_hdd`. В томе `hot` находится один SSD-диск (`fast_ssd`), а также задается ограничение на максимальный размер куска, который может храниться на этом томе (1GB). Все куски такой таблицы больше 1GB будут записываться сразу на том `cold`, в котором содержится один HDD-диск `disk1`. Также при заполнении диска `fast_ssd` более чем на 80% данные будут переноситься на диск `disk1` фоновым процессом. Порядок томов в политиках хранения важен, при достижении условий на переполнение тома данные переносятся на следующий. Порядок дисков в томах так же важен, данные пишутся по очереди на каждый из них. diff --git a/docs/ru/interfaces/third-party/client-libraries.md b/docs/ru/interfaces/third-party/client-libraries.md index f55bbe2a47d..a4659e9ac4e 100644 --- a/docs/ru/interfaces/third-party/client-libraries.md +++ b/docs/ru/interfaces/third-party/client-libraries.md @@ -8,6 +8,7 @@ sidebar_label: "Клиентские библиотеки от сторонни :::danger "Disclaimer" Яндекс не поддерживает перечисленные ниже библиотеки и не проводит тщательного тестирования для проверки их качества. +::: - Python: - [infi.clickhouse_orm](https://github.com/Infinidat/infi.clickhouse_orm) diff --git a/docs/ru/interfaces/third-party/gui.md b/docs/ru/interfaces/third-party/gui.md index c34b799347b..e7190362dc4 100644 --- a/docs/ru/interfaces/third-party/gui.md +++ b/docs/ru/interfaces/third-party/gui.md @@ -177,19 +177,20 @@ sidebar_label: "Визуальные интерфейсы от сторонни ### Yandex DataLens {#yandex-datalens} -[Yandex DataLens](https://cloud.yandex.ru/services/datalens) — cервис визуализации и анализа данных. +[Yandex DataLens](https://datalens.yandex.ru) — cервис визуализации и анализа данных. Основные возможности: - Широкий выбор инструментов визуализации, от простых столбчатых диаграмм до сложных дашбордов. - Возможность опубликовать дашборды на широкую аудиторию. - Поддержка множества источников данных, включая ClickHouse. -- Хранение материализованных данных в кластере ClickHouse DataLens. -Для небольших проектов DataLens [доступен бесплатно](https://cloud.yandex.ru/docs/datalens/pricing), в том числе и для коммерческого использования. +DataLens [доступен бесплатно](https://cloud.yandex.ru/docs/datalens/pricing), в том числе и для коммерческого использования. +- [Знакомство с DataLens]((https://youtu.be/57ngi_6BINE). +- [Чат сообщества DataLens](https://t.me/YandexDataLens) - [Документация DataLens](https://cloud.yandex.ru/docs/datalens/). -- [Пособие по визуализации данных из ClickHouse](https://cloud.yandex.ru/docs/solutions/datalens/data-from-ch-visualization). +- [Сценарий по визуализации данных из ClickHouse](https://cloud.yandex.ru/docs/solutions/datalens/data-from-ch-visualization). ### Holistics Software {#holistics-software} diff --git a/docs/ru/operations/clickhouse-keeper.md b/docs/ru/operations/clickhouse-keeper.md index 67be83e13b2..3a931529b32 100644 --- a/docs/ru/operations/clickhouse-keeper.md +++ b/docs/ru/operations/clickhouse-keeper.md @@ -325,21 +325,21 @@ clickhouse-keeper-converter --zookeeper-logs-dir /var/lib/zookeeper/version-2 -- Например, для кластера из 3 нод, алгоритм кворума продолжает работать при отказе не более чем одной ноды. Конфигурация кластера может быть изменена динамически с некоторыми ограничениями. -Переконфигурация также использует Raft, поэтому для добавление новой ноды кластера или исключения старой ноды из него требуется достижения кворума в рамках текущей конфигурации кластера. +Переконфигурация также использует Raft, поэтому для добавления новой ноды кластера или исключения старой ноды требуется достижение кворума в рамках текущей конфигурации кластера. Если в вашем кластере произошел отказ большего числа нод, чем допускает Raft для вашей текущей конфигурации и у вас нет возможности восстановить их работоспособность, Raft перестанет работать и не позволит изменить конфигурацию стандартным механизмом. -Тем не менее ClickHousr Keeper имеет возможность запуститься в режиме восстановления, который позволяет переконфигурировать класте используя только одну ноду кластера. +Тем не менее ClickHouse Keeper имеет возможность запуститься в режиме восстановления, который позволяет переконфигурировать кластер используя только одну ноду кластера. Этот механизм может использоваться только как крайняя мера, когда вы не можете восстановить существующие ноды кластера или запустить новый сервер с тем же идентификатором. Важно: - Удостоверьтесь, что отказавшие ноды не смогут в дальнейшем подключиться к кластеру в будущем. -- Не запускайте новые ноды, пока не завешите процедуру ниже. +- Не запускайте новые ноды, пока не завершите процедуру ниже. После того, как выполнили действия выше выполните следующие шаги. -1. Выберете одну ноду Keeper, которая станет новым лидером. Учтите, что данные которые с этой ноды будут испольщзованы всем кластером, поэтому рекомендуется выбрать ноду с наиболее актуальным состоянием. +1. Выберете одну ноду Keeper, которая станет новым лидером. Учтите, что данные с этой ноды будут использованы всем кластером, поэтому рекомендуется выбрать ноду с наиболее актуальным состоянием. 2. Перед дальнейшими действиям сделайте резервную копию данных из директорий `log_storage_path` и `snapshot_storage_path`. 3. Измените настройки на всех нодах кластера, которые вы собираетесь использовать. -4. Отправьте команду `rcvr` на ноду, которую вы выбрали или остановите ее и запустите заново с аргументом `--force-recovery`. Это переведет ноду в режим восстановления. +4. Отправьте команду `rcvr` на ноду, которую вы выбрали, или остановите ее и запустите заново с аргументом `--force-recovery`. Это переведет ноду в режим восстановления. 5. Запускайте остальные ноды кластера по одной и проверяйте, что команда `mntr` возвращает `follower` в выводе состояния `zk_server_state` перед тем, как запустить следующую ноду. -6. Пока нода работает в режиме восстановления, лидер будет возвращать ошибку на запрос `mntr` пока кворум не будет достигнут с помощью новых нод. Любые запросы от клиентов и постедователей будут возвращать ошибку. +6. Пока нода работает в режиме восстановления, лидер будет возвращать ошибку на запрос `mntr` пока кворум не будет достигнут с помощью новых нод. Любые запросы от клиентов и последователей будут возвращать ошибку. 7. После достижения кворума лидер перейдет в нормальный режим работы и станет обрабатывать все запросы через Raft. Удостоверьтесь, что запрос `mntr` возвращает `leader` в выводе состояния `zk_server_state`. diff --git a/docs/ru/operations/opentelemetry.md b/docs/ru/operations/opentelemetry.md index b6c5e89bcc6..4e127e9e0f0 100644 --- a/docs/ru/operations/opentelemetry.md +++ b/docs/ru/operations/opentelemetry.md @@ -10,6 +10,7 @@ ClickHouse поддерживает [OpenTelemetry](https://opentelemetry.io/) :::danger "Предупреждение" Поддержка стандарта экспериментальная и будет со временем меняться. +::: ## Обеспечение поддержки контекста трассировки в ClickHouse diff --git a/docs/ru/operations/server-configuration-parameters/settings.md b/docs/ru/operations/server-configuration-parameters/settings.md index bffa3c39a60..e29b9def9d4 100644 --- a/docs/ru/operations/server-configuration-parameters/settings.md +++ b/docs/ru/operations/server-configuration-parameters/settings.md @@ -26,6 +26,7 @@ ClickHouse перезагружает встроенные словари с з :::danger "Внимание" Лучше не использовать, если вы только начали работать с ClickHouse. +::: Общий вид конфигурации: @@ -1064,6 +1065,7 @@ ClickHouse использует потоки из глобального пул :::danger "Обратите внимание" Завершающий слеш обязателен. +::: **Пример** @@ -1330,6 +1332,7 @@ TCP порт для защищённого обмена данными с кли :::danger "Обратите внимание" Завершающий слеш обязателен. +::: **Пример** diff --git a/docs/ru/operations/storing-data.md b/docs/ru/operations/storing-data.md index 2f5c9c95ea4..56081c82bc9 100644 --- a/docs/ru/operations/storing-data.md +++ b/docs/ru/operations/storing-data.md @@ -82,7 +82,7 @@ sidebar_label: "Хранение данных на внешних дисках" - `type` — `encrypted`. Иначе зашифрованный диск создан не будет. - `disk` — тип диска для хранения данных. -- `key` — ключ для шифрования и расшифровки. Тип: [Uint64](../sql-reference/data-types/int-uint.md). Вы можете использовать параметр `key_hex` для шифрования в шестнадцатеричной форме. +- `key` — ключ для шифрования и расшифровки. Тип: [UInt64](../sql-reference/data-types/int-uint.md). Вы можете использовать параметр `key_hex` для шифрования в шестнадцатеричной форме. Вы можете указать несколько ключей, используя атрибут `id` (смотрите пример выше). Необязательные параметры: diff --git a/docs/ru/sql-reference/data-types/aggregatefunction.md b/docs/ru/sql-reference/data-types/aggregatefunction.md index 21b452acb1d..e42b467e4af 100644 --- a/docs/ru/sql-reference/data-types/aggregatefunction.md +++ b/docs/ru/sql-reference/data-types/aggregatefunction.md @@ -6,7 +6,7 @@ sidebar_label: AggregateFunction # AggregateFunction {#data-type-aggregatefunction} -Агрегатные функции могут обладать определяемым реализацией промежуточным состоянием, которое может быть сериализовано в тип данных, соответствующий AggregateFunction(…), и быть записано в таблицу обычно посредством [материализованного представления] (../../sql-reference/statements/create/view.md). Чтобы получить промежуточное состояние, обычно используются агрегатные функции с суффиксом `-State`. Чтобы в дальнейшем получить агрегированные данные необходимо использовать те же агрегатные функции с суффиксом `-Merge`. +Агрегатные функции могут обладать определяемым реализацией промежуточным состоянием, которое может быть сериализовано в тип данных, соответствующий AggregateFunction(…), и быть записано в таблицу обычно посредством [материализованного представления](../../sql-reference/statements/create/view.md). Чтобы получить промежуточное состояние, обычно используются агрегатные функции с суффиксом `-State`. Чтобы в дальнейшем получить агрегированные данные необходимо использовать те же агрегатные функции с суффиксом `-Merge`. `AggregateFunction(name, types_of_arguments…)` — параметрический тип данных. diff --git a/docs/ru/sql-reference/data-types/geo.md b/docs/ru/sql-reference/data-types/geo.md index a7c5f79b0be..24d981195f5 100644 --- a/docs/ru/sql-reference/data-types/geo.md +++ b/docs/ru/sql-reference/data-types/geo.md @@ -10,6 +10,7 @@ ClickHouse поддерживает типы данных для отображ :::danger "Предупреждение" Сейчас использование типов данных для работы с географическими структурами является экспериментальной возможностью. Чтобы использовать эти типы данных, включите настройку `allow_experimental_geo_types = 1`. +::: **См. также** - [Хранение географических структур данных](https://ru.wikipedia.org/wiki/GeoJSON). diff --git a/docs/ru/sql-reference/data-types/special-data-types/interval.md b/docs/ru/sql-reference/data-types/special-data-types/interval.md index 856275ed8f2..109ceee7852 100644 --- a/docs/ru/sql-reference/data-types/special-data-types/interval.md +++ b/docs/ru/sql-reference/data-types/special-data-types/interval.md @@ -10,6 +10,7 @@ sidebar_label: Interval :::danger "Внимание" Нельзя использовать типы данных `Interval` для хранения данных в таблице. +::: Структура: diff --git a/docs/ru/sql-reference/data-types/tuple.md b/docs/ru/sql-reference/data-types/tuple.md index 76370d01c0d..8953134d154 100644 --- a/docs/ru/sql-reference/data-types/tuple.md +++ b/docs/ru/sql-reference/data-types/tuple.md @@ -34,7 +34,7 @@ SELECT tuple(1,'a') AS x, toTypeName(x) ## Особенности работы с типами данных {#osobennosti-raboty-s-tipami-dannykh} -При создании кортежа «на лету» ClickHouse автоматически определяет тип каждого аргументов как минимальный из типов, который может сохранить значение аргумента. Если аргумент — [NULL](../../sql-reference/data-types/tuple.md#null-literal), то тип элемента кортежа — [Nullable](nullable.md). +При создании кортежа «на лету» ClickHouse автоматически определяет тип всех аргументов как минимальный из типов, который может сохранить значение аргумента. Если аргумент — [NULL](../../sql-reference/data-types/tuple.md#null-literal), то тип элемента кортежа — [Nullable](nullable.md). Пример автоматического определения типа данных: diff --git a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-polygon.md b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-polygon.md index 64637edc4a4..24f29d3bf53 100644 --- a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-polygon.md +++ b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-polygon.md @@ -61,7 +61,7 @@ LAYOUT(POLYGON(STORE_POLYGON_KEY_COLUMN 1)) - Мультиполигон. Представляет из себя массив полигонов. Каждый полигон задается двумерным массивом точек — первый элемент этого массива задает внешнюю границу полигона, последующие элементы могут задавать дырки, вырезаемые из него. -Точки могут задаваться массивом или кортежем из своих координат. В текущей реализации поддерживается только двумерные точки. +Точки могут задаваться массивом или кортежем из своих координат. В текущей реализации поддерживаются только двумерные точки. Пользователь может [загружать свои собственные данные](../../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md) во всех поддерживаемых ClickHouse форматах. @@ -80,7 +80,7 @@ LAYOUT(POLYGON(STORE_POLYGON_KEY_COLUMN 1)) - `POLYGON`. Синоним к `POLYGON_INDEX_CELL`. Запросы к словарю осуществляются с помощью стандартных [функций](../../../sql-reference/functions/ext-dict-functions.md) для работы со внешними словарями. -Важным отличием является то, что здесь ключами будут являются точки, для которых хочется найти содержащий их полигон. +Важным отличием является то, что здесь ключами являются точки, для которых хочется найти содержащий их полигон. **Пример** diff --git a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md index 8c01b8295bf..a711287ae8e 100644 --- a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md +++ b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md @@ -59,6 +59,7 @@ ClickHouse поддерживает следующие виды ключей: :::danger "Обратите внимание" Ключ не надо дополнительно описывать в атрибутах. +::: ### Числовой ключ {#ext_dict-numeric-key} diff --git a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts.md b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts.md index 314fefab5eb..a262a354889 100644 --- a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts.md +++ b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts.md @@ -14,7 +14,7 @@ ClickHouse: - Периодически обновляет их и динамически подгружает отсутствующие значения. - Позволяет создавать внешние словари с помощью xml-файлов или [DDL-запросов](../../statements/create/dictionary.md#create-dictionary-query). -Конфигурация внешних словарей может находится в одном или нескольких xml-файлах. Путь к конфигурации указывается в параметре [dictionaries_config](../../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-dictionaries_config). +Конфигурация внешних словарей может находиться в одном или нескольких xml-файлах. Путь к конфигурации указывается в параметре [dictionaries_config](../../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-dictionaries_config). Словари могут загружаться при старте сервера или при первом использовании, в зависимости от настройки [dictionaries_lazy_load](../../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-dictionaries_lazy_load). diff --git a/docs/ru/sql-reference/functions/introspection.md b/docs/ru/sql-reference/functions/introspection.md index 7d04dff6b72..26497ef21d3 100644 --- a/docs/ru/sql-reference/functions/introspection.md +++ b/docs/ru/sql-reference/functions/introspection.md @@ -22,7 +22,7 @@ sidebar_label: "Функции интроспекции" ClickHouse сохраняет отчеты профилировщика в [журнал трассировки](../../operations/system-tables/trace_log.md#system_tables-trace_log) в системной таблице. Убедитесь, что таблица и профилировщик настроены правильно. -## addresssToLine {#addresstoline} +## addressToLine {#addresstoline} Преобразует адрес виртуальной памяти внутри процесса сервера ClickHouse в имя файла и номер строки в исходном коде ClickHouse. diff --git a/docs/ru/sql-reference/operators/exists.md b/docs/ru/sql-reference/operators/exists.md index 3fc085fe021..38855abbcf3 100644 --- a/docs/ru/sql-reference/operators/exists.md +++ b/docs/ru/sql-reference/operators/exists.md @@ -8,7 +8,8 @@ slug: /ru/sql-reference/operators/exists `EXISTS` может быть использован в секции [WHERE](../../sql-reference/statements/select/where.md). :::danger "Предупреждение" - Ссылки на таблицы или столбцы основного запроса не поддерживаются в подзапросе. + Ссылки на таблицы или столбцы основного запроса не поддерживаются в подзапросе. +::: **Синтаксис** diff --git a/docs/ru/sql-reference/operators/in.md b/docs/ru/sql-reference/operators/in.md index fa679b890a7..60400fb2b31 100644 --- a/docs/ru/sql-reference/operators/in.md +++ b/docs/ru/sql-reference/operators/in.md @@ -38,9 +38,9 @@ SELECT '1' IN (SELECT 1); └──────────────────────┘ ``` -Если в качестве правой части оператора указано имя таблицы (например, `UserID IN users`), то это эквивалентно подзапросу `UserID IN (SELECT * FROM users)`. Это используется при работе с внешними данными, отправляемым вместе с запросом. Например, вместе с запросом может быть отправлено множество идентификаторов посетителей, загруженное во временную таблицу users, по которому следует выполнить фильтрацию. +Если в качестве правой части оператора указано имя таблицы (например, `UserID IN users`), то это эквивалентно подзапросу `UserID IN (SELECT * FROM users)`. Это используется при работе с внешними данными, отправляемыми вместе с запросом. Например, вместе с запросом может быть отправлено множество идентификаторов посетителей, загруженное во временную таблицу users, по которому следует выполнить фильтрацию. -Если в качестве правой части оператора, указано имя таблицы, имеющий движок Set (подготовленное множество, постоянно находящееся в оперативке), то множество не будет создаваться заново при каждом запросе. +Если в качестве правой части оператора, указано имя таблицы, имеющей движок Set (подготовленное множество, постоянно находящееся в оперативке), то множество не будет создаваться заново при каждом запросе. В подзапросе может быть указано более одного столбца для фильтрации кортежей. Пример: @@ -49,9 +49,9 @@ SELECT '1' IN (SELECT 1); SELECT (CounterID, UserID) IN (SELECT CounterID, UserID FROM ...) FROM ... ``` -Типы столбцов слева и справа оператора IN, должны совпадать. +Типы столбцов слева и справа оператора IN должны совпадать. -Оператор IN и подзапрос могут встречаться в любой части запроса, в том числе в агрегатных и лямбда функциях. +Оператор IN и подзапрос могут встречаться в любой части запроса, в том числе в агрегатных и лямбда-функциях. Пример: ``` sql @@ -122,7 +122,7 @@ FROM t_null Существует два варианта IN-ов с подзапросами (аналогично для JOIN-ов): обычный `IN` / `JOIN` и `GLOBAL IN` / `GLOBAL JOIN`. Они отличаются способом выполнения при распределённой обработке запроса. -:::note "Attention" +:::note "Внимание" Помните, что алгоритмы, описанные ниже, могут работать иначе в зависимости от [настройки](../../operations/settings/settings.md) `distributed_product_mode`. ::: При использовании обычного IN-а, запрос отправляется на удалённые серверы, и на каждом из них выполняются подзапросы в секциях `IN` / `JOIN`. @@ -228,7 +228,7 @@ SELECT CounterID, count() FROM distributed_table_1 WHERE UserID IN (SELECT UserI SETTINGS max_parallel_replicas=3 ``` -преобразуются на каждом сервере в +преобразуется на каждом сервере в ```sql SELECT CounterID, count() FROM local_table_1 WHERE UserID IN (SELECT UserID FROM local_table_2 WHERE CounterID < 100) diff --git a/docs/ru/sql-reference/operators/index.md b/docs/ru/sql-reference/operators/index.md index 57c426cb5ad..b5fec3cb38c 100644 --- a/docs/ru/sql-reference/operators/index.md +++ b/docs/ru/sql-reference/operators/index.md @@ -263,6 +263,7 @@ SELECT toDateTime('2014-10-26 00:00:00', 'Europe/Moscow') AS time, time + 60 * 6 │ 2014-10-26 00:00:00 │ 2014-10-26 23:00:00 │ 2014-10-27 00:00:00 │ └─────────────────────┴─────────────────────┴─────────────────────┘ ``` +::: **Смотрите также** diff --git a/docs/ru/sql-reference/statements/alter/view.md b/docs/ru/sql-reference/statements/alter/view.md index 2d4823bba3a..e6f6730ff99 100644 --- a/docs/ru/sql-reference/statements/alter/view.md +++ b/docs/ru/sql-reference/statements/alter/view.md @@ -6,7 +6,7 @@ sidebar_label: VIEW # Выражение ALTER TABLE … MODIFY QUERY {#alter-modify-query} -Вы можеие изменить запрос `SELECT`, который был задан при создании [материализованного представления](../create/view.md#materialized), с помощью запроса 'ALTER TABLE … MODIFY QUERY'. Используйте его если при создании материализованного представления не использовалась секция `TO [db.]name`. Настройка `allow_experimental_alter_materialized_view_structure` должна быть включена. +Вы можете изменить запрос `SELECT`, который был задан при создании [материализованного представления](../create/view.md#materialized), с помощью запроса 'ALTER TABLE … MODIFY QUERY'. Используйте его если при создании материализованного представления не использовалась секция `TO [db.]name`. Настройка `allow_experimental_alter_materialized_view_structure` должна быть включена. Если при создании материализованного представления использовалась конструкция `TO [db.]name`, то для изменения отсоедините представление с помощью [DETACH](../detach.md), измените таблицу с помощью [ALTER TABLE](index.md), а затем снова присоедините запрос с помощью [ATTACH](../attach.md). diff --git a/docs/ru/sql-reference/statements/create/table.md b/docs/ru/sql-reference/statements/create/table.md index 7a930b529ed..64eae49be6c 100644 --- a/docs/ru/sql-reference/statements/create/table.md +++ b/docs/ru/sql-reference/statements/create/table.md @@ -260,8 +260,8 @@ ENGINE = MergeTree() Кодеки шифрования: -- `CODEC('AES-128-GCM-SIV')` — Зашифровывает данные с помощью AES-128 в режиме [RFC 8452](https://tools.ietf.org/html/rfc8452) GCM-SIV. -- `CODEC('AES-256-GCM-SIV')` — Зашифровывает данные с помощью AES-256 в режиме GCM-SIV. +- `CODEC('AES-128-GCM-SIV')` — Зашифровывает данные с помощью AES-128 в режиме [RFC 8452](https://tools.ietf.org/html/rfc8452) GCM-SIV. +- `CODEC('AES-256-GCM-SIV')` — Зашифровывает данные с помощью AES-256 в режиме GCM-SIV. Эти кодеки используют фиксированный одноразовый ключ шифрования. Таким образом, это детерминированное шифрование. Оно совместимо с поддерживающими дедупликацию движками, в частности, [ReplicatedMergeTree](../../../engines/table-engines/mergetree-family/replication.md). Однако у шифрования имеется недостаток: если дважды зашифровать один и тот же блок данных, текст на выходе получится одинаковым, и злоумышленник, у которого есть доступ к диску, заметит эту эквивалентность (при этом доступа к содержимому он не получит). @@ -274,10 +274,10 @@ ENGINE = MergeTree() **Пример** ```sql -CREATE TABLE mytable +CREATE TABLE mytable ( x String Codec(AES_128_GCM_SIV) -) +) ENGINE = MergeTree ORDER BY x; ``` @@ -287,10 +287,10 @@ ENGINE = MergeTree ORDER BY x; **Пример** ```sql -CREATE TABLE mytable +CREATE TABLE mytable ( x String Codec(Delta, LZ4, AES_128_GCM_SIV) -) +) ENGINE = MergeTree ORDER BY x; ``` @@ -299,7 +299,7 @@ ENGINE = MergeTree ORDER BY x; ClickHouse поддерживает временные таблицы со следующими характеристиками: - Временные таблицы исчезают после завершения сессии, в том числе при обрыве соединения. -- Временная таблица использует только модуль памяти. +- Временная таблица использует движок таблиц Memory когда движок не указан и она может использовать любой движок таблиц за исключением движков Replicated и `KeeperMap`. - Невозможно указать базу данных для временной таблицы. Она создается вне баз данных. - Невозможно создать временную таблицу распределённым DDL запросом на всех серверах кластера (с опцией `ON CLUSTER`): такая таблица существует только в рамках существующей сессии. - Если временная таблица имеет то же имя, что и некоторая другая, то, при упоминании в запросе без указания БД, будет использована временная таблица. @@ -313,7 +313,7 @@ CREATE TEMPORARY TABLE [IF NOT EXISTS] table_name name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1], name2 [type2] [DEFAULT|MATERIALIZED|ALIAS expr2], ... -) +) [ENGINE = engine] ``` В большинстве случаев, временные таблицы создаются не вручную, а при использовании внешних данных для запроса, или при распределённом `(GLOBAL) IN`. Подробнее см. соответствующие разделы diff --git a/docs/ru/sql-reference/statements/grant.md b/docs/ru/sql-reference/statements/grant.md index 7c281634c98..73c63850750 100644 --- a/docs/ru/sql-reference/statements/grant.md +++ b/docs/ru/sql-reference/statements/grant.md @@ -107,7 +107,8 @@ GRANT SELECT(x,y) ON db.table TO john WITH GRANT OPTION - [CREATE](#grant-create) - `CREATE DATABASE` - `CREATE TABLE` - - `CREATE TEMPORARY TABLE` + - `CREATE ARBITRARY TEMPORARY TABLE` + - `CREATE TEMPORARY TABLE` - `CREATE VIEW` - `CREATE DICTIONARY` - `CREATE FUNCTION` @@ -314,7 +315,8 @@ GRANT INSERT(x,y) ON db.table TO john - `CREATE`. Уровень: `GROUP` - `CREATE DATABASE`. Уровень: `DATABASE` - `CREATE TABLE`. Уровень: `TABLE` - - `CREATE TEMPORARY TABLE`. Уровень: `GLOBAL` + - `CREATE ARBITRARY TEMPORARY TABLE`. Уровень: `GLOBAL` + - `CREATE TEMPORARY TABLE`. Уровень: `GLOBAL` - `CREATE VIEW`. Уровень: `VIEW` - `CREATE DICTIONARY`. Уровень: `DICTIONARY` diff --git a/docs/ru/sql-reference/statements/optimize.md b/docs/ru/sql-reference/statements/optimize.md index b70bba2d765..26993183232 100644 --- a/docs/ru/sql-reference/statements/optimize.md +++ b/docs/ru/sql-reference/statements/optimize.md @@ -10,6 +10,7 @@ sidebar_label: OPTIMIZE :::danger "Внимание" `OPTIMIZE` не устраняет причину появления ошибки `Too many parts`. +::: **Синтаксис** diff --git a/docs/zh/engines/database-engines/replicated.md b/docs/zh/engines/database-engines/replicated.md index df5872e9b19..843d7a3edd5 100644 --- a/docs/zh/engines/database-engines/replicated.md +++ b/docs/zh/engines/database-engines/replicated.md @@ -19,7 +19,6 @@ CREATE DATABASE testdb ENGINE = Replicated('zoo_path', 'shard_name', 'replica_na - `shard_name` — 分片的名字。数据库副本按`shard_name`分组到分片中。 - `replica_name` — 副本的名字。同一分片的所有副本的副本名称必须不同。 -!!! note "警告" 对于[ReplicatedMergeTree](../table-engines/mergetree-family/replication.md#table_engines-replication)表，如果没有提供参数，则使用默认参数:`/clickhouse/tables/{uuid}/{shard}`和`{replica}`。这些可以在服务器设置[default_replica_path](../../operations/server-configuration-parameters/settings.md#default_replica_path)和[default_replica_name](../../operations/server-configuration-parameters/settings.md#default_replica_name)中更改。宏`{uuid}`被展开到表的uuid， `{shard}`和`{replica}`被展开到服务器配置的值，而不是数据库引擎参数。但是在将来，可以使用Replicated数据库的`shard_name`和`replica_name`。 ## 使用方式 {#specifics-and-recommendations} @@ -52,8 +51,8 @@ CREATE TABLE r.rmt (n UInt64) ENGINE=ReplicatedMergeTree ORDER BY n; ``` ``` text -┌─────hosts────────────┬──status─┬─error─┬─num_hosts_remaining─┬─num_hosts_active─┐ -│ shard1|replica1 │ 0 │ │ 2 │ 0 │ +┌─────hosts────────────┬──status─┬─error─┬─num_hosts_remaining─┬─num_hosts_active─┐ +│ shard1|replica1 │ 0 │ │ 2 │ 0 │ │ shard1|other_replica │ 0 │ │ 1 │ 0 │ │ other_shard|r1 │ 0 │ │ 0 │ 0 │ └──────────────────────┴─────────┴───────┴─────────────────────┴──────────────────┘ @@ -62,13 +61,13 @@ CREATE TABLE r.rmt (n UInt64) ENGINE=ReplicatedMergeTree ORDER BY n; 显示系统表: ``` sql -SELECT cluster, shard_num, replica_num, host_name, host_address, port, is_local +SELECT cluster, shard_num, replica_num, host_name, host_address, port, is_local FROM system.clusters WHERE cluster='r'; ``` ``` text -┌─cluster─┬─shard_num─┬─replica_num─┬─host_name─┬─host_address─┬─port─┬─is_local─┐ -│ r │ 1 │ 1 │ node3 │ 127.0.0.1 │ 9002 │ 0 │ +┌─cluster─┬─shard_num─┬─replica_num─┬─host_name─┬─host_address─┬─port─┬─is_local─┐ +│ r │ 1 │ 1 │ node3 │ 127.0.0.1 │ 9002 │ 0 │ │ r │ 2 │ 1 │ node2 │ 127.0.0.1 │ 9001 │ 0 │ │ r │ 2 │ 2 │ node1 │ 127.0.0.1 │ 9000 │ 1 │ └─────────┴───────────┴─────────────┴───────────┴──────────────┴──────┴──────────┘ @@ -83,9 +82,9 @@ node1 :) SELECT materialize(hostName()) AS host, groupArray(n) FROM r.d GROUP BY ``` ``` text -┌─hosts─┬─groupArray(n)─┐ -│ node1 │ [1,3,5,7,9] │ -│ node2 │ [0,2,4,6,8] │ +┌─hosts─┬─groupArray(n)─┐ +│ node1 │ [1,3,5,7,9] │ +│ node2 │ [0,2,4,6,8] │ └───────┴───────────────┘ ``` @@ -98,8 +97,8 @@ node4 :) CREATE DATABASE r ENGINE=Replicated('some/path/r','other_shard','r2'); 集群配置如下所示: ``` text -┌─cluster─┬─shard_num─┬─replica_num─┬─host_name─┬─host_address─┬─port─┬─is_local─┐ -│ r │ 1 │ 1 │ node3 │ 127.0.0.1 │ 9002 │ 0 │ +┌─cluster─┬─shard_num─┬─replica_num─┬─host_name─┬─host_address─┬─port─┬─is_local─┐ +│ r │ 1 │ 1 │ node3 │ 127.0.0.1 │ 9002 │ 0 │ │ r │ 1 │ 2 │ node4 │ 127.0.0.1 │ 9003 │ 0 │ │ r │ 2 │ 1 │ node2 │ 127.0.0.1 │ 9001 │ 0 │ │ r │ 2 │ 2 │ node1 │ 127.0.0.1 │ 9000 │ 1 │ @@ -113,8 +112,8 @@ node2 :) SELECT materialize(hostName()) AS host, groupArray(n) FROM r.d GROUP BY ``` ```text -┌─hosts─┬─groupArray(n)─┐ -│ node2 │ [1,3,5,7,9] │ -│ node4 │ [0,2,4,6,8] │ +┌─hosts─┬─groupArray(n)─┐ +│ node2 │ [1,3,5,7,9] │ +│ node4 │ [0,2,4,6,8] │ └───────┴───────────────┘ ``` diff --git a/docs/zh/engines/table-engines/mergetree-family/mergetree.md b/docs/zh/engines/table-engines/mergetree-family/mergetree.md index 40aa764e2d3..1fcf64fcd25 100644 --- a/docs/zh/engines/table-engines/mergetree-family/mergetree.md +++ b/docs/zh/engines/table-engines/mergetree-family/mergetree.md @@ -1,6 +1,6 @@ ---- -slug: /zh/engines/table-engines/mergetree-family/mergetree ---- +--- +slug: /zh/engines/table-engines/mergetree-family/mergetree +--- # MergeTree {#table_engines-mergetree} Clickhouse 中最强大的表引擎当属 `MergeTree` （合并树）引擎及该系列（`*MergeTree`）中的其他引擎。 @@ -25,8 +25,9 @@ Clickhouse 中最强大的表引擎当属 `MergeTree` （合并树）引擎及 需要的话，您可以给表设置一个采样方法。 -!!! note "注意" - [合并](../special/merge.md#merge) 引擎并不属于 `*MergeTree` 系列。 +:::info +[合并](../special/merge.md#merge) 引擎并不属于 `*MergeTree` 系列。 +::: ## 建表 {#table_engine-mergetree-creating-a-table} @@ -364,7 +365,7 @@ WHERE 子句中的条件可以包含对某列数据进行运算的函数表达 常量参数小于 ngram 大小的函数不能使用 `ngrambf_v1` 进行查询优化。 -!!! note "注意" +:::note 布隆过滤器可能会包含不符合条件的匹配，所以 `ngrambf_v1`, `tokenbf_v1` 和 `bloom_filter` 索引不能用于结果返回为假的函数，例如： - 可以用来优化的场景 @@ -379,6 +380,7 @@ WHERE 子句中的条件可以包含对某列数据进行运算的函数表达 - `NOT s = 1` - `s != 1` - `NOT startsWith(s, 'test')` +::: ## 并发数据访问 {#concurrent-data-access} diff --git a/docs/zh/engines/table-engines/special/distributed.md b/docs/zh/engines/table-engines/special/distributed.md index c832e9e19ba..3c8a7a833d0 100644 --- a/docs/zh/engines/table-engines/special/distributed.md +++ b/docs/zh/engines/table-engines/special/distributed.md @@ -45,7 +45,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] AS [db2.]name2 - [insert_distributed_sync](../../../operations/settings/settings.md#insert_distributed_sync) 设置 - [MergeTree](../../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-multiple-volumes) 查看示例 - + **分布式设置** - `fsync_after_insert` - 对异步插入到分布式的文件数据执行`fsync`。确保操作系统将所有插入的数据刷新到启动节点**磁盘上的一个文件**中。 @@ -66,19 +66,20 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] AS [db2.]name2 - `monitor_max_sleep_time_ms` - 等同于 [distributed_directory_monitor_max_sleep_time_ms](../../../operations/settings/settings.md#distributed_directory_monitor_max_sleep_time_ms) -!!! note "备注" +::note +**稳定性设置** (`fsync_...`): - **稳定性设置** (`fsync_...`): +- 只影响异步插入(例如:`insert_distributed_sync=false`), 当数据首先存储在启动节点磁盘上，然后再异步发送到shard。 +— 可能会显著降低`insert`的性能 +- 影响将存储在分布式表文件夹中的数据写入 **接受您插入的节点** 。如果你需要保证写入数据到底层的MergeTree表中，请参阅 `system.merge_tree_settings` 中的持久性设置(`...fsync...`) - - 只影响异步插入(例如:`insert_distributed_sync=false`), 当数据首先存储在启动节点磁盘上，然后再异步发送到shard。 - — 可能会显著降低`insert`的性能 - - 影响将存储在分布式表文件夹中的数据写入 **接受您插入的节点** 。如果你需要保证写入数据到底层的MergeTree表中，请参阅 `system.merge_tree_settings` 中的持久性设置(`...fsync...`) +**插入限制设置** (`..._insert`) 请见: - **插入限制设置** (`..._insert`) 请见: +- [insert_distributed_sync](../../../operations/settings/settings.md#insert_distributed_sync) 设置 +- [prefer_localhost_replica](../../../operations/settings/settings.md#settings-prefer-localhost-replica) 设置 +- `bytes_to_throw_insert` 在 `bytes_to_delay_insert` 之前处理，所以你不应该设置它的值小于 `bytes_to_delay_insert` +::: - - [insert_distributed_sync](../../../operations/settings/settings.md#insert_distributed_sync) 设置 - - [prefer_localhost_replica](../../../operations/settings/settings.md#settings-prefer-localhost-replica) 设置 - - `bytes_to_throw_insert` 在 `bytes_to_delay_insert` 之前处理，所以你不应该设置它的值小于 `bytes_to_delay_insert` **示例** ``` sql @@ -214,7 +215,7 @@ SELECT 查询会被发送到所有分片，并且无论数据在分片中如何 ## 读取数据 {#distributed-reading-data} - + 当查询一个`Distributed`表时，`SELECT`查询被发送到所有的分片，不管数据是如何分布在分片上的(它们可以完全随机分布)。当您添加一个新分片时，您不必将旧数据传输到它。相反，您可以使用更重的权重向其写入新数据——数据的分布会稍微不均匀，但查询将正确有效地工作。 当启用`max_parallel_replicas`选项时，查询处理将在单个分片中的所有副本之间并行化。更多信息，请参见[max_parallel_replicas](../../../operations/settings/settings.md#settings-max_parallel_replicas)。 @@ -225,8 +226,9 @@ SELECT 查询会被发送到所有分片，并且无论数据在分片中如何 - `_shard_num` — 表`system.clusters` 中的 `shard_num` 值 . 数据类型: [UInt32](../../../sql-reference/data-types/int-uint.md). -!!! note "备注" - 因为 [remote](../../../sql-reference/table-functions/remote.md) 和 [cluster](../../../sql-reference/table-functions/cluster.mdx) 表方法内部创建了分布式表， `_shard_num` 对他们都有效. +:::note +因为 [remote](../../../sql-reference/table-functions/remote.md) 和 [cluster](../../../sql-reference/table-functions/cluster.mdx) 表方法内部创建了分布式表， `_shard_num` 对他们都有效. +::: **详见** - [虚拟列](../../../engines/table-engines/index.md#table_engines-virtual_columns) 描述 diff --git a/docs/zh/getting-started/tutorial.md b/docs/zh/getting-started/tutorial.md index bef3ecee35f..989cf5f57d8 100644 --- a/docs/zh/getting-started/tutorial.md +++ b/docs/zh/getting-started/tutorial.md @@ -617,8 +617,9 @@ INSERT INTO tutorial.hits_all SELECT * FROM tutorial.hits_v1; 启用本机复制[Zookeeper](http://zookeeper.apache.org/)是必需的。 ClickHouse负责所有副本的数据一致性，并在失败后自动运行恢复过程。建议将ZooKeeper集群部署在单独的服务器上（其中没有其他进程，包括运行的ClickHouse）。 -!!! note "注意" - ZooKeeper不是一个严格的要求：在某些简单的情况下，您可以通过将数据写入应用程序代码中的所有副本来复制数据。 这种方法是**不**建议的，在这种情况下，ClickHouse将无法保证所有副本上的数据一致性。 因此需要由您的应用来保证这一点。 +:::note +ZooKeeper不是一个严格的要求：在某些简单的情况下，您可以通过将数据写入应用程序代码中的所有副本来复制数据。 这种方法是**不**建议的，在这种情况下，ClickHouse将无法保证所有副本上的数据一致性。 因此需要由您的应用来保证这一点。 +::: ZooKeeper位置在配置文件中指定: diff --git a/docs/zh/guides/improving-query-performance/skipping-indexes.md b/docs/zh/guides/improving-query-performance/skipping-indexes.md index 2f9ce09d77f..f5889898c2c 100644 --- a/docs/zh/guides/improving-query-performance/skipping-indexes.md +++ b/docs/zh/guides/improving-query-performance/skipping-indexes.md @@ -1,6 +1,6 @@ --- slug: /zh/guides/improving-query-performance/skipping-indexes -sidebar_label: Data Skipping Indexes +sidebar_label: 跳数索引 sidebar_position: 2 --- diff --git a/docs/zh/guides/improving-query-performance/sparse-primary-indexes.md b/docs/zh/guides/improving-query-performance/sparse-primary-indexes.md index e773a02fbc3..18b23a79f86 100644 --- a/docs/zh/guides/improving-query-performance/sparse-primary-indexes.md +++ b/docs/zh/guides/improving-query-performance/sparse-primary-indexes.md @@ -1,6 +1,6 @@ --- slug: /zh/guides/improving-query-performance/sparse-primary-indexes -sidebar_label: Sparse Primary Indexes +sidebar_label: 主键稀疏索引 sidebar_position: 20 --- diff --git a/docs/zh/interfaces/formats.md b/docs/zh/interfaces/formats.md index 852b327366b..fd3cf743818 100644 --- a/docs/zh/interfaces/formats.md +++ b/docs/zh/interfaces/formats.md @@ -685,8 +685,9 @@ CREATE TABLE IF NOT EXISTS example_table - 如果`input_format_defaults_for_omitted_fields = 0`, 那么`x`和`a`的默认值等于`0`(作为`UInt32`数据类型的默认值)。 - 如果`input_format_defaults_for_omitted_fields = 1`, 那么`x`的默认值为`0`，但`a`的默认值为`x * 2`。 -!!! note "注意" +:::warning 当使用`input_format_defaults_for_omitted_fields = 1`插入数据时，与使用`input_format_defaults_for_omitted_fields = 0`相比，ClickHouse消耗更多的计算资源。 +::: ### Selecting Data {#selecting-data} @@ -708,8 +709,9 @@ CREATE TABLE IF NOT EXISTS example_table 与[JSON](#json)格式不同，没有替换无效的UTF-8序列。值以与`JSON`相同的方式转义。 -!!! note "提示" +:::info 字符串中可以输出任意一组字节。如果您确信表中的数据可以被格式化为JSON而不会丢失任何信息，那么就使用`JSONEachRow`格式。 +::: ### Nested Structures {#jsoneachrow-nested} @@ -1216,9 +1218,9 @@ SET format_avro_schema_registry_url = 'http://schema-registry'; SELECT * FROM topic1_stream; ``` -!!! note "警告" - 设置 `format_avro_schema_registry_url` 需要写入配置文件`users.xml`以在Clickhouse重启后，该设置仍为您的设定值。您也可以在使用Kafka引擎的时候指定该设置。 - +:::warning +设置 `format_avro_schema_registry_url` 需要写入配置文件`users.xml`以在Clickhouse重启后，该设置仍为您的设定值。您也可以在使用Kafka引擎的时候指定该设置。 +::: ## Parquet {#data-format-parquet} diff --git a/docs/zh/interfaces/http.md b/docs/zh/interfaces/http.md index 11754ed3e2f..e0c12193a6a 100644 --- a/docs/zh/interfaces/http.md +++ b/docs/zh/interfaces/http.md @@ -188,8 +188,9 @@ $ curl -vsS "http://localhost:8123/?enable_http_compression=1" -d 'SELECT number $ echo "SELECT 1" | gzip -c | curl -sS --data-binary @- -H 'Content-Encoding: gzip' 'http://localhost:8123/' ``` -!!! note "警告" - 一些HTTP客户端可能会在默认情况下从服务器解压数据(使用`gzip`和`deflate`)，即使您未正确地使用了压缩设置，您也可能会得到解压数据。 +:::warning +一些HTTP客户端可能会在默认情况下从服务器解压数据(使用`gzip`和`deflate`)，即使您未正确地使用了压缩设置，您也可能会得到解压数据。 +::: 您可以使用`database`URL参数或`X-ClickHouse-Database`头来指定默认数据库。 @@ -447,8 +448,9 @@ $ curl -H 'XXX:TEST_HEADER_VALUE' -H 'PARAMS_XXX:max_threads' 'http://localhost: max_final_threads 2 ``` -!!! note "警告" - 在一个`predefined_query_handler`中，只支持insert类型的一个`查询`。 +:::warning +在一个`predefined_query_handler`中，只支持insert类型的一个`查询`。 +::: ### 动态查询 {#dynamic_query_handler} diff --git a/docs/zh/interfaces/third-party/index.md b/docs/zh/interfaces/third-party/index.md index 0c6b194d04a..64c0b30a9c0 100644 --- a/docs/zh/interfaces/third-party/index.md +++ b/docs/zh/interfaces/third-party/index.md @@ -13,5 +13,6 @@ sidebar_position: 24 - [GUI](../../interfaces/third-party/gui.md) - [Proxies](../../interfaces/third-party/proxy.md) -!!! note "注意" +:::note 支持通用API的通用工具[ODBC](../../interfaces/odbc.md)或[JDBC](../../interfaces/jdbc.md)，通常也适用于ClickHouse，但这里没有列出，因为它们实在太多了。 +::: diff --git a/docs/zh/operations/access-rights.md b/docs/zh/operations/access-rights.md index 713cf991f47..8d640ef32c0 100644 --- a/docs/zh/operations/access-rights.md +++ b/docs/zh/operations/access-rights.md @@ -24,9 +24,9 @@ ClickHouse权限实体包括： 我们建议你使用SQL工作流的方式。当然配置的方式也可以同时起作用, 所以如果你正在用服务端配置的方式来管理权限和账户，你可以平滑的切换到SQL驱动的工作流方式。 -!!! note "警告" - 你无法同时使用两个配置的方式来管理同一个权限实体。 - +:::warning +你无法同时使用两个配置的方式来管理同一个权限实体。 +::: ## 用法 {#access-control-usage} diff --git a/docs/zh/operations/server-configuration-parameters/settings.md b/docs/zh/operations/server-configuration-parameters/settings.md index 89ae411b9e0..2fd04b1260a 100644 --- a/docs/zh/operations/server-configuration-parameters/settings.md +++ b/docs/zh/operations/server-configuration-parameters/settings.md @@ -528,8 +528,9 @@ SSL客户端/服务器配置。 包含数据的目录的路径。 -!!! note "注" - 尾部斜杠是强制性的。 +:::note +尾部斜杠是强制性的。 +::: **示例** @@ -714,8 +715,9 @@ TCP端口，用于与客户端进行安全通信。 使用它与 [OpenSSL](#serv 用于处理大型查询的临时数据的路径。 -!!! note "注" - 尾部斜杠是强制性的。 +:::note +尾部斜杠是强制性的。 +::: **示例** @@ -728,11 +730,12 @@ TCP端口，用于与客户端进行安全通信。 使用它与 [OpenSSL](#serv 从政策 [`storage_configuration`](../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-multiple-volumes) 存储临时文件。 如果没有设置 [`tmp_path`](#server-settings-tmp_path) 被使用，否则被忽略。 -!!! note "注" - - `move_factor` 被忽略 +:::note +- `move_factor` 被忽略 - `keep_free_space_bytes` 被忽略 - `max_data_part_size_bytes` 被忽略 -您必须在该政策中只有一个卷 +::: ## uncompressed_cache_size {#server-settings-uncompressed_cache_size} diff --git a/docs/zh/operations/settings/settings-profiles.md b/docs/zh/operations/settings/settings-profiles.md index 1ad394950bf..5051276607f 100644 --- a/docs/zh/operations/settings/settings-profiles.md +++ b/docs/zh/operations/settings/settings-profiles.md @@ -8,8 +8,9 @@ sidebar_label: "\u8BBE\u7F6E\u914D\u7F6E" 设置配置是设置的集合，并按照相同的名称进行分组。 -!!! note "信息" - ClickHouse 还支持用 [SQL驱动的工作流](../../operations/access-rights.md#access-control) 管理设置配置。我们建议使用它。 +:::info +ClickHouse 还支持用 [SQL驱动的工作流](../../operations/access-rights.md#access-control) 管理设置配置。我们建议使用它。 +::: 设置配置可以任意命名。你可以为不同的用户指定相同的设置配置。您可以在设置配置中写入的最重要的内容是 `readonly=1`，这将确保只读访问。 diff --git a/docs/zh/operations/settings/settings-users.md b/docs/zh/operations/settings/settings-users.md index de4aa27df69..3fb97bbddb2 100644 --- a/docs/zh/operations/settings/settings-users.md +++ b/docs/zh/operations/settings/settings-users.md @@ -10,8 +10,9 @@ sidebar_label: "\u7528\u6237\u8BBE\u7F6E" `user.xml` 中的 `users` 配置段包含了用户配置 -!!! note "提示" - ClickHouse还支持 [SQL驱动的工作流](../access-rights.md#access-control) 用于管理用户。 我们建议使用它。 +:::note +ClickHouse还支持 [SQL驱动的工作流](../access-rights.md#access-control) 用于管理用户。 我们建议使用它。 +::: `users` 配置段的结构: diff --git a/docs/zh/operations/settings/settings.md b/docs/zh/operations/settings/settings.md index 4107a499463..457b208602f 100644 --- a/docs/zh/operations/settings/settings.md +++ b/docs/zh/operations/settings/settings.md @@ -266,8 +266,9 @@ INSERT INTO test VALUES (lower('Hello')), (lower('world')), (lower('INSERT')), ( 执行时 `INSERT` 查询时，将省略的输入列值替换为相应列的默认值。 此选项仅适用于 [JSONEachRow](../../interfaces/formats.md#jsoneachrow), [CSV](../../interfaces/formats.md#csv) 和 [TabSeparated](../../interfaces/formats.md#tabseparated) 格式。 -!!! note "注" - 启用此选项后，扩展表元数据将从服务器发送到客户端。 它会消耗服务器上的额外计算资源，并可能降低性能。 +:::note +启用此选项后，扩展表元数据将从服务器发送到客户端。 它会消耗服务器上的额外计算资源，并可能降低性能。 +::: 可能的值: diff --git a/docs/zh/operations/system-tables/parts.md b/docs/zh/operations/system-tables/parts.md index 0bd728f543f..0ebac3944ff 100644 --- a/docs/zh/operations/system-tables/parts.md +++ b/docs/zh/operations/system-tables/parts.md @@ -99,8 +99,9 @@ slug: /zh/operations/system-tables/parts - `move_ttl_info.expression` ([Array](../../sql-reference/data-types/array.md)([String](../../sql-reference/data-types/string.md))) — 表达式的数组。 每个表达式定义一个 [TTL MOVE 规则](../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-ttl). - !!! note "警告" - 保留 `move_ttl_info.expression` 数组主要是为了向后兼容，现在检查 `TTL MOVE` 规则最简单的方法是使用 `move_ttl_info.min` 和 `move_ttl_info.max` 字段。 +:::warning +保留 `move_ttl_info.expression` 数组主要是为了向后兼容，现在检查 `TTL MOVE` 规则最简单的方法是使用 `move_ttl_info.min` 和 `move_ttl_info.max` 字段。 +::: - `move_ttl_info.min` ([Array](../../sql-reference/data-types/array.md)([DateTime](../../sql-reference/data-types/datetime.md))) — 日期值和时间值的数组。数组中的每个元素都描述了一个 [TTL MOVE rule](../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-ttl) 的最小键值。 diff --git a/docs/zh/operations/system-tables/query_log.md b/docs/zh/operations/system-tables/query_log.md index 93e5771d4b5..7149282dfcc 100644 --- a/docs/zh/operations/system-tables/query_log.md +++ b/docs/zh/operations/system-tables/query_log.md @@ -8,8 +8,9 @@ machine_translated_rev: 5decc73b5dc60054f19087d3690c4eb99446a6c3 包含已执行查询的相关信息，例如：开始时间、处理持续时间、错误消息。 -!!! note "注" - 此表不包含以下内容的摄取数据 `INSERT` 查询。 +:::note +此表不包含以下内容的摄取数据 `INSERT` 查询。 +::: 您可以更改query_log的设置，在服务器配置的 [query_log](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-query-log) 部分。 diff --git a/docs/zh/sql-reference/aggregate-functions/reference/corr.md b/docs/zh/sql-reference/aggregate-functions/reference/corr.md index 01a89e428ab..48b5bf904f5 100644 --- a/docs/zh/sql-reference/aggregate-functions/reference/corr.md +++ b/docs/zh/sql-reference/aggregate-functions/reference/corr.md @@ -12,5 +12,6 @@ sidebar_position: 107 计算Pearson相关系数: `Σ((x - x̅)(y - y̅)) / sqrt(Σ((x - x̅)^2) * Σ((y - y̅)^2))`。 -!!! note "注" - 该函数使用数值不稳定的算法。 如果你需要 [数值稳定性](https://en.wikipedia.org/wiki/Numerical_stability) 在计算中，使用 `corrStable` 函数。 它的工作速度较慢，但提供较低的计算错误。 +:::note +该函数使用数值不稳定的算法。 如果你需要 [数值稳定性](https://en.wikipedia.org/wiki/Numerical_stability) 在计算中，使用 `corrStable` 函数。 它的工作速度较慢，但提供较低的计算错误。 +::: diff --git a/docs/zh/sql-reference/aggregate-functions/reference/covarpop.md b/docs/zh/sql-reference/aggregate-functions/reference/covarpop.md index 93bfee15684..e98270dc896 100644 --- a/docs/zh/sql-reference/aggregate-functions/reference/covarpop.md +++ b/docs/zh/sql-reference/aggregate-functions/reference/covarpop.md @@ -12,5 +12,6 @@ covarPop(x, y) 计算 `Σ((x - x̅)(y - y̅)) / n` 的值。 -!!! note "注" - 该函数使用数值不稳定的算法。 如果你需要 [数值稳定性](https://en.wikipedia.org/wiki/Numerical_stability) 在计算中，使用 `covarPopStable` 函数。 它的工作速度较慢，但提供了较低的计算错误。 +:::note +该函数使用数值不稳定的算法。 如果你需要 [数值稳定性](https://en.wikipedia.org/wiki/Numerical_stability) 在计算中，使用 `covarPopStable` 函数。 它的工作速度较慢，但提供了较低的计算错误。 +::: diff --git a/docs/zh/sql-reference/aggregate-functions/reference/covarsamp.md b/docs/zh/sql-reference/aggregate-functions/reference/covarsamp.md index 7c8565211b1..1b8dfc4a60f 100644 --- a/docs/zh/sql-reference/aggregate-functions/reference/covarsamp.md +++ b/docs/zh/sql-reference/aggregate-functions/reference/covarsamp.md @@ -14,5 +14,6 @@ covarSamp(x, y) 返回Float64。 当 `n <= 1`, 返回 +∞。 -!!! note "注" - 该函数使用数值不稳定的算法。 如果你需要 [数值稳定性](https://en.wikipedia.org/wiki/Numerical_stability) 在计算中，使用 `covarSampStable` 函数。 它的工作速度较慢，但提供较低的计算错误。 +:::note +该函数使用数值不稳定的算法。 如果你需要 [数值稳定性](https://en.wikipedia.org/wiki/Numerical_stability) 在计算中，使用 `covarSampStable` 函数。 它的工作速度较慢，但提供较低的计算错误。 +::: diff --git a/docs/zh/sql-reference/aggregate-functions/reference/quantiletiming.md b/docs/zh/sql-reference/aggregate-functions/reference/quantiletiming.md index 5e14ce6a11c..af7ad77c717 100644 --- a/docs/zh/sql-reference/aggregate-functions/reference/quantiletiming.md +++ b/docs/zh/sql-reference/aggregate-functions/reference/quantiletiming.md @@ -37,8 +37,9 @@ quantileTiming(level)(expr) 否则，计算结果将四舍五入到16毫秒的最接近倍数。 -!!! note "注" - 对于计算页面加载时间分位数， 此函数比[quantile](../../../sql-reference/aggregate-functions/reference/quantile.md#quantile)更有效和准确。 +:::note +对于计算页面加载时间分位数， 此函数比[quantile](../../../sql-reference/aggregate-functions/reference/quantile.md#quantile)更有效和准确。 +::: **返回值** @@ -46,8 +47,9 @@ quantileTiming(level)(expr) 类型: `Float32`。 -!!! note "注" - 如果没有值传递给函数（当使用 `quantileTimingIf`), [NaN](../../../sql-reference/data-types/float.md#data_type-float-nan-inf)被返回。 这样做的目的是将这些案例与导致零的案例区分开来。 参见 [ORDER BY clause](../../../sql-reference/statements/select/order-by.md#select-order-by) 对于 `NaN` 值排序注意事项。 +:::note +如果没有值传递给函数（当使用 `quantileTimingIf`), [NaN](../../../sql-reference/data-types/float.md#data_type-float-nan-inf)被返回。 这样做的目的是将这些案例与导致零的案例区分开来。 参见 [ORDER BY clause](../../../sql-reference/statements/select/order-by.md#select-order-by) 对于 `NaN` 值排序注意事项。 +::: **示例** diff --git a/docs/zh/sql-reference/aggregate-functions/reference/quantiletimingweighted.md b/docs/zh/sql-reference/aggregate-functions/reference/quantiletimingweighted.md index 2c28583343a..b520a0f96af 100644 --- a/docs/zh/sql-reference/aggregate-functions/reference/quantiletimingweighted.md +++ b/docs/zh/sql-reference/aggregate-functions/reference/quantiletimingweighted.md @@ -39,8 +39,9 @@ quantileTimingWeighted(level)(expr, weight) 否则，计算结果将四舍五入到16毫秒的最接近倍数。 -!!! note "注" - 对于计算页面加载时间分位数， 此函数比[quantile](../../../sql-reference/aggregate-functions/reference/quantile.md#quantile)更有效和准确。 +:::note +对于计算页面加载时间分位数， 此函数比[quantile](../../../sql-reference/aggregate-functions/reference/quantile.md#quantile)更有效和准确。 +::: **返回值** @@ -48,8 +49,9 @@ quantileTimingWeighted(level)(expr, weight) 类型: `Float32`。 -!!! note "注" - 如果没有值传递给函数（当使用 `quantileTimingIf`), [NaN](../../../sql-reference/data-types/float.md#data_type-float-nan-inf)被返回。 这样做的目的是将这些案例与导致零的案例区分开来。 参见 [ORDER BY clause](../../../sql-reference/statements/select/order-by.md#select-order-by) 对于 `NaN` 值排序注意事项。 +:::note +如果没有值传递给函数（当使用 `quantileTimingIf`), [NaN](../../../sql-reference/data-types/float.md#data_type-float-nan-inf)被返回。 这样做的目的是将这些案例与导致零的案例区分开来。 参见 [ORDER BY clause](../../../sql-reference/statements/select/order-by.md#select-order-by) 对于 `NaN` 值排序注意事项。 +::: **示例** diff --git a/docs/zh/sql-reference/aggregate-functions/reference/stddevpop.md b/docs/zh/sql-reference/aggregate-functions/reference/stddevpop.md index ea82e21e46f..a113084cdee 100644 --- a/docs/zh/sql-reference/aggregate-functions/reference/stddevpop.md +++ b/docs/zh/sql-reference/aggregate-functions/reference/stddevpop.md @@ -7,5 +7,6 @@ sidebar_position: 30 结果等于 [varPop](../../../sql-reference/aggregate-functions/reference/varpop.md)的平方根。 -!!! note "注" - 该函数使用数值不稳定的算法。 如果你需要 [数值稳定性](https://en.wikipedia.org/wiki/Numerical_stability) 在计算中，使用 `stddevPopStable` 函数。 它的工作速度较慢，但提供较低的计算错误。 +:::note +该函数使用数值不稳定的算法。 如果你需要 [数值稳定性](https://en.wikipedia.org/wiki/Numerical_stability) 在计算中，使用 `stddevPopStable` 函数。 它的工作速度较慢，但提供较低的计算错误。 +::: diff --git a/docs/zh/sql-reference/aggregate-functions/reference/stddevsamp.md b/docs/zh/sql-reference/aggregate-functions/reference/stddevsamp.md index efeafb71072..d242f4e3401 100644 --- a/docs/zh/sql-reference/aggregate-functions/reference/stddevsamp.md +++ b/docs/zh/sql-reference/aggregate-functions/reference/stddevsamp.md @@ -7,5 +7,6 @@ sidebar_position: 31 结果等于 [varSamp] (../../../sql-reference/aggregate-functions/reference/varsamp.md)的平方根。 -!!! note "注" - 该函数使用数值不稳定的算法。 如果你需要 [数值稳定性](https://en.wikipedia.org/wiki/Numerical_stability) 在计算中，使用 `stddevSampStable` 函数。 它的工作速度较慢，但提供较低的计算错误。 +:::note +该函数使用数值不稳定的算法。 如果你需要 [数值稳定性](https://en.wikipedia.org/wiki/Numerical_stability) 在计算中，使用 `stddevSampStable` 函数。 它的工作速度较慢，但提供较低的计算错误。 +::: \ No newline at end of file diff --git a/docs/zh/sql-reference/aggregate-functions/reference/uniqcombined.md b/docs/zh/sql-reference/aggregate-functions/reference/uniqcombined.md index edc790ec00f..7b4a78d662e 100644 --- a/docs/zh/sql-reference/aggregate-functions/reference/uniqcombined.md +++ b/docs/zh/sql-reference/aggregate-functions/reference/uniqcombined.md @@ -36,8 +36,9 @@ uniqCombined(HLL_precision)(x[, ...]) - 确定性地提供结果（它不依赖于查询处理顺序）。 -!!! note "注" - 由于它对非 `String` 类型使用32位哈希，对于基数显著大于`UINT_MAX` ，结果将有非常高的误差(误差将在几百亿不同值之后迅速提高), 因此这种情况，你应该使用 [uniqCombined64](../../../sql-reference/aggregate-functions/reference/uniqcombined64.md#agg_function-uniqcombined64) +:::note +由于它对非 `String` 类型使用32位哈希，对于基数显著大于`UINT_MAX` ，结果将有非常高的误差(误差将在几百亿不同值之后迅速提高), 因此这种情况，你应该使用 [uniqCombined64](../../../sql-reference/aggregate-functions/reference/uniqcombined64.md#agg_function-uniqcombined64) +::: 相比于 [uniq](../../../sql-reference/aggregate-functions/reference/uniq.md#agg_function-uniq) 函数, 该 `uniqCombined`: diff --git a/docs/zh/sql-reference/aggregate-functions/reference/varpop.md b/docs/zh/sql-reference/aggregate-functions/reference/varpop.md index eb17955210b..6d6b0acc615 100644 --- a/docs/zh/sql-reference/aggregate-functions/reference/varpop.md +++ b/docs/zh/sql-reference/aggregate-functions/reference/varpop.md @@ -9,5 +9,6 @@ sidebar_position: 32 换句话说，计算一组数据的离差。 返回 `Float64`。 -!!! note "注" - 该函数使用数值不稳定的算法。 如果你需要 [数值稳定性](https://en.wikipedia.org/wiki/Numerical_stability) 在计算中，使用 `varPopStable` 函数。 它的工作速度较慢，但提供较低的计算错误。 +:::note +该函数使用数值不稳定的算法。 如果你需要 [数值稳定性](https://en.wikipedia.org/wiki/Numerical_stability) 在计算中，使用 `varPopStable` 函数。 它的工作速度较慢，但提供较低的计算错误。 +::: diff --git a/docs/zh/sql-reference/aggregate-functions/reference/varsamp.md b/docs/zh/sql-reference/aggregate-functions/reference/varsamp.md index 9b9d0ced92d..508e35445a7 100644 --- a/docs/zh/sql-reference/aggregate-functions/reference/varsamp.md +++ b/docs/zh/sql-reference/aggregate-functions/reference/varsamp.md @@ -11,5 +11,6 @@ sidebar_position: 33 返回 `Float64`。 当 `n <= 1`，返回 `+∞`。 -!!! note "注" - 该函数使用数值不稳定的算法。 如果你需要 [数值稳定性](https://en.wikipedia.org/wiki/Numerical_stability) 在计算中，使用 `varSampStable` 函数。 它的工作速度较慢，但提供较低的计算错误。 +:::note +该函数使用数值不稳定的算法。 如果你需要 [数值稳定性](https://en.wikipedia.org/wiki/Numerical_stability) 在计算中，使用 `varSampStable` 函数。 它的工作速度较慢，但提供较低的计算错误。 +::: diff --git a/docs/zh/sql-reference/ansi.md b/docs/zh/sql-reference/ansi.md index cdccee0084f..74f13256eba 100644 --- a/docs/zh/sql-reference/ansi.md +++ b/docs/zh/sql-reference/ansi.md @@ -6,8 +6,9 @@ sidebar_label: "ANSI\u517C\u5BB9\u6027" # ClickHouse SQL方言 与ANSI SQL的兼容性{#ansi-sql-compatibility-of-clickhouse-sql-dialect} -!!! note "注" - 本文参考Annex G所著的[ISO/IEC CD 9075-2:2011](https://www.iso.org/obp/ui/#iso:std:iso-iec:9075:-2:ed-4:v1:en:sec:8)标准. +:::note +本文参考Annex G所著的[ISO/IEC CD 9075-2:2011](https://www.iso.org/obp/ui/#iso:std:iso-iec:9075:-2:ed-4:v1:en:sec:8)标准. +::: ## 行为差异 {#differences-in-behaviour} diff --git a/docs/zh/sql-reference/data-types/simpleaggregatefunction.md b/docs/zh/sql-reference/data-types/simpleaggregatefunction.md index b26994a775e..601cb602a78 100644 --- a/docs/zh/sql-reference/data-types/simpleaggregatefunction.md +++ b/docs/zh/sql-reference/data-types/simpleaggregatefunction.md @@ -25,10 +25,10 @@ slug: /zh/sql-reference/data-types/simpleaggregatefunction - [`argMax`](../../sql-reference/aggregate-functions/reference/argmax.md) -!!! note "注" - `SimpleAggregateFunction(func, Type)` 的值外观和存储方式于 `Type` 相同, 所以你不需要应用带有 `-Merge`/`-State` 后缀的函数。 - - `SimpleAggregateFunction` 的性能优于具有相同聚合函数的 `AggregateFunction` 。 +:::note +`SimpleAggregateFunction(func, Type)` 的值外观和存储方式于 `Type` 相同, 所以你不需要应用带有 `-Merge`/`-State` 后缀的函数。 +`SimpleAggregateFunction` 的性能优于具有相同聚合函数的 `AggregateFunction` 。 +::: **参数** diff --git a/docs/zh/sql-reference/functions/string-search-functions.md b/docs/zh/sql-reference/functions/string-search-functions.md index 756ac7c16c7..e4167127424 100644 --- a/docs/zh/sql-reference/functions/string-search-functions.md +++ b/docs/zh/sql-reference/functions/string-search-functions.md @@ -42,8 +42,9 @@ slug: /zh/sql-reference/functions/string-search-functions 对于不区分大小写的搜索或/和UTF-8格式，使用函数`multiSearchAnyCaseInsensitive，multiSearchAnyUTF8，multiSearchAnyCaseInsensitiveUTF8`。 -!!! note "注意" - 在所有`multiSearch*`函数中，由于实现规范，needles的数量应小于28。 +:::note +在所有`multiSearch*`函数中，由于实现规范，needles的数量应小于28。 +::: ## 匹配（大海捞针，模式) {#matchhaystack-pattern} @@ -60,8 +61,9 @@ slug: /zh/sql-reference/functions/string-search-functions 与`match`相同，但如果所有正则表达式都不匹配，则返回0；如果任何模式匹配，则返回1。它使用[超扫描](https://github.com/intel/hyperscan)库。对于在字符串中搜索子字符串的模式，最好使用«multisearchany»，因为它更高效。 -!!! note "注意" - 任何`haystack`字符串的长度必须小于232\字节，否则抛出异常。这种限制是因为hyperscan API而产生的。 +:::note +任何`haystack`字符串的长度必须小于232\字节，否则抛出异常。这种限制是因为hyperscan API而产生的。 +::: ## multiMatchAnyIndex（大海捞针，\[模式1，模式2, …, patternn\]) {#multimatchanyindexhaystack-pattern1-pattern2-patternn} @@ -75,11 +77,13 @@ slug: /zh/sql-reference/functions/string-search-functions 与`multiFuzzyMatchAny`相同，但返回匹配项的匹配能容的索引位置。 -!!! note "注意" - `multiFuzzyMatch*`函数不支持UTF-8正则表达式，由于hyperscan限制，这些表达式被按字节解析。 +:::note +`multiFuzzyMatch*`函数不支持UTF-8正则表达式，由于hyperscan限制，这些表达式被按字节解析。 +::: -!!! note "注意" - 如要关闭所有hyperscan函数的使用，请设置`SET allow_hyperscan = 0;`。 +:::note +如要关闭所有hyperscan函数的使用，请设置`SET allow_hyperscan = 0;`。 +::: ## 提取（大海捞针，图案) {#extracthaystack-pattern} @@ -119,5 +123,6 @@ slug: /zh/sql-reference/functions/string-search-functions 对于不区分大小写的搜索或/和UTF-8格式，使用函数`ngramSearchCaseInsensitive，ngramSearchUTF8，ngramSearchCaseInsensitiveUTF8`。 -!!! note "注意" - 对于UTF-8，我们使用3-gram。所有这些都不是完全公平的n-gram距离。我们使用2字节哈希来散列n-gram，然后计算这些哈希表之间的（非）对称差异 - 可能会发生冲突。对于UTF-8不区分大小写的格式，我们不使用公平的`tolower`函数 - 我们将每个Unicode字符字节的第5位（从零开始）和字节的第一位归零 - 这适用于拉丁语，主要用于所有西里尔字母。 +:::note +对于UTF-8，我们使用3-gram。所有这些都不是完全公平的n-gram距离。我们使用2字节哈希来散列n-gram，然后计算这些哈希表之间的（非）对称差异 - 可能会发生冲突。对于UTF-8不区分大小写的格式，我们不使用公平的`tolower`函数 - 我们将每个Unicode字符字节的第5位（从零开始）和字节的第一位归零 - 这适用于拉丁语，主要用于所有西里尔字母。 +::: diff --git a/docs/zh/sql-reference/statements/alter/delete.md b/docs/zh/sql-reference/statements/alter/delete.md index 85d3d3077a7..5eb77c35a93 100644 --- a/docs/zh/sql-reference/statements/alter/delete.md +++ b/docs/zh/sql-reference/statements/alter/delete.md @@ -12,8 +12,9 @@ ALTER TABLE [db.]table [ON CLUSTER cluster] DELETE WHERE filter_expr 删除匹配指定过滤表达式的数据。实现为[突变](../../../sql-reference/statements/alter/index.md#mutations). -!!! note "备注" - `ALTER TABLE`前缀使得这个语法不同于大多数其他支持SQL的系统。它的目的是表示，与OLTP数据库中的类似查询不同，这是一个不为经常使用而设计的繁重操作。 +:::note +`ALTER TABLE`前缀使得这个语法不同于大多数其他支持SQL的系统。它的目的是表示，与OLTP数据库中的类似查询不同，这是一个不为经常使用而设计的繁重操作。 +::: `filter_expr` 的类型必须是`UInt8`。该查询删除表中该表达式接受非零值的行。 diff --git a/docs/zh/sql-reference/statements/alter/index.md b/docs/zh/sql-reference/statements/alter/index.md index b0f0fc21cbe..8320b207725 100644 --- a/docs/zh/sql-reference/statements/alter/index.md +++ b/docs/zh/sql-reference/statements/alter/index.md @@ -17,8 +17,9 @@ sidebar_label: ALTER - [CONSTRAINT](../../../sql-reference/statements/alter/constraint.md) - [TTL](../../../sql-reference/statements/alter/ttl.md) -!!! note "备注" - 大多数 `ALTER TABLE` 查询只支持[\*MergeTree](../../../engines/table-engines/mergetree-family/index.md)表，以及[Merge](../../../engines/table-engines/special/merge.md)和[Distributed](../../../engines/table-engines/special/distributed.md)。 +:::note +大多数 `ALTER TABLE` 查询只支持[\*MergeTree](../../../engines/table-engines/mergetree-family/index.md)表，以及[Merge](../../../engines/table-engines/special/merge.md)和[Distributed](../../../engines/table-engines/special/distributed.md)。 +::: 这些 `ALTER` 语句操作视图: diff --git a/docs/zh/sql-reference/statements/alter/order-by.md b/docs/zh/sql-reference/statements/alter/order-by.md index e70a8b59c85..e50c4e6e805 100644 --- a/docs/zh/sql-reference/statements/alter/order-by.md +++ b/docs/zh/sql-reference/statements/alter/order-by.md @@ -14,5 +14,6 @@ ALTER TABLE [db].name [ON CLUSTER cluster] MODIFY ORDER BY new_expression 从某种意义上说，该命令是轻量级的，它只更改元数据。要保持数据部分行按排序键表达式排序的属性，您不能向排序键添加包含现有列的表达式(仅在相同的`ALTER`查询中由`ADD COLUMN`命令添加的列，没有默认的列值)。 -!!! note "备注" - 它只适用于[`MergeTree`](../../../engines/table-engines/mergetree-family/mergetree.md)表族(包括[replicated](../../../engines/table-engines/mergetree-family/replication.md)表)。 +:::note +它只适用于[`MergeTree`](../../../engines/table-engines/mergetree-family/mergetree.md)表族(包括[replicated](../../../engines/table-engines/mergetree-family/replication.md)表)。 +::: diff --git a/docs/zh/sql-reference/statements/alter/setting.md b/docs/zh/sql-reference/statements/alter/setting.md index e2d597554e7..2e1e97db331 100644 --- a/docs/zh/sql-reference/statements/alter/setting.md +++ b/docs/zh/sql-reference/statements/alter/setting.md @@ -14,8 +14,9 @@ sidebar_label: SETTING ALTER TABLE [db].name [ON CLUSTER cluster] MODIFY|RESET SETTING ... ``` -!!! note "注意" - 这些查询只能应用于 [MergeTree](../../../engines/table-engines/mergetree-family/mergetree.md) 表。 +:::note +这些查询只能应用于 [MergeTree](../../../engines/table-engines/mergetree-family/mergetree.md) 表。 +::: ## 修改设置 {#alter_modify_setting} diff --git a/docs/zh/sql-reference/statements/alter/update.md b/docs/zh/sql-reference/statements/alter/update.md index 522eb0a705b..97b2b43d889 100644 --- a/docs/zh/sql-reference/statements/alter/update.md +++ b/docs/zh/sql-reference/statements/alter/update.md @@ -12,8 +12,9 @@ ALTER TABLE [db.]table UPDATE column1 = expr1 [, ...] WHERE filter_expr 操作与指定过滤表达式相匹配的数据。作为一个[变更 mutation](../../../sql-reference/statements/alter/index.md#mutations)来实现. -!!! note "Note" - `ALTER TABLE` 的前缀使这个语法与其他大多数支持SQL的系统不同。它的目的是表明，与OLTP数据库中的类似查询不同，这是一个繁重的操作，不是为频繁使用而设计。 +:::note +`ALTER TABLE` 的前缀使这个语法与其他大多数支持SQL的系统不同。它的目的是表明，与OLTP数据库中的类似查询不同，这是一个繁重的操作，不是为频繁使用而设计。 +::: `filter_expr`必须是`UInt8`类型。这个查询将指定列的值更新为行中相应表达式的值，对于这些行，`filter_expr`取值为非零。使用`CAST`操作符将数值映射到列的类型上。不支持更新用于计算主键或分区键的列。 diff --git a/docs/zh/sql-reference/statements/exchange.md b/docs/zh/sql-reference/statements/exchange.md index e6ac1dbf1dc..47cefa0d2e6 100644 --- a/docs/zh/sql-reference/statements/exchange.md +++ b/docs/zh/sql-reference/statements/exchange.md @@ -9,8 +9,9 @@ sidebar_label: EXCHANGE 以原子方式交换两个表或字典的名称。 此任务也可以通过使用[RENAME](./rename.md)来完成，但在这种情况下操作不是原子的。 -!!! note "注意" +:::note `EXCHANGE`仅支持[Atomic](../../engines/database-engines/atomic.md)数据库引擎. +::: **语法** diff --git a/docs/zh/sql-reference/statements/rename.md b/docs/zh/sql-reference/statements/rename.md index c26dce306cc..156306fbd3e 100644 --- a/docs/zh/sql-reference/statements/rename.md +++ b/docs/zh/sql-reference/statements/rename.md @@ -9,8 +9,9 @@ sidebar_label: RENAME 重命名数据库、表或字典。 可以在单个查询中重命名多个实体。 请注意，具有多个实体的`RENAME`查询是非原子操作。 要以原子方式交换实体名称，请使用[EXCHANGE](./exchange.md)语法. -!!! note "注意" +:::note `RENAME`仅支持[Atomic](../../engines/database-engines/atomic.md)数据库引擎. +::: **语法** diff --git a/docs/zh/sql-reference/statements/select/group-by.md b/docs/zh/sql-reference/statements/select/group-by.md index 31c1649bc30..29c72ce7e45 100644 --- a/docs/zh/sql-reference/statements/select/group-by.md +++ b/docs/zh/sql-reference/statements/select/group-by.md @@ -11,8 +11,9 @@ sidebar_label: GROUP BY - 在所有的表达式在 [SELECT](../../../sql-reference/statements/select/index.md), [HAVING](../../../sql-reference/statements/select/having)，和 [ORDER BY](../../../sql-reference/statements/select/order-by.md) 子句中 **必须** 基于键表达式进行计算 **或** 上 [聚合函数](../../../sql-reference/aggregate-functions/index.md) 在非键表达式（包括纯列）上。 换句话说，从表中选择的每个列必须用于键表达式或聚合函数内，但不能同时使用。 - 聚合结果 `SELECT` 查询将包含尽可能多的行，因为有唯一值 “grouping key” 在源表中。 通常这会显着减少行数，通常是数量级，但不一定：如果所有行数保持不变 “grouping key” 值是不同的。 -!!! note "注" - 还有一种额外的方法可以在表上运行聚合。 如果查询仅在聚合函数中包含表列，则 `GROUP BY` 可以省略，并且通过一个空的键集合来假定聚合。 这样的查询总是只返回一行。 +:::note +还有一种额外的方法可以在表上运行聚合。 如果查询仅在聚合函数中包含表列，则 `GROUP BY` 可以省略，并且通过一个空的键集合来假定聚合。 这样的查询总是只返回一行。 +::: ## 空处理 {#null-processing} diff --git a/docs/zh/sql-reference/statements/select/join.md b/docs/zh/sql-reference/statements/select/join.md index 08290a02de5..a2686aa5e53 100644 --- a/docs/zh/sql-reference/statements/select/join.md +++ b/docs/zh/sql-reference/statements/select/join.md @@ -39,8 +39,9 @@ ClickHouse中提供的其他联接类型: ## 严格 {#join-settings} -!!! note "注" - 可以使用以下方式复盖默认的严格性值 [join_default_strictness](../../../operations/settings/settings.md#settings-join_default_strictness) 设置。 +:::note +可以使用以下方式复盖默认的严格性值 [join_default_strictness](../../../operations/settings/settings.md#settings-join_default_strictness) 设置。 +::: Also the behavior of ClickHouse server for `ANY JOIN` operations depends on the [any_join_distinct_right_table_keys](../../../operations/settings/settings.md#any_join_distinct_right_table_keys) setting. @@ -91,8 +92,9 @@ USING (equi_column1, ... equi_columnN, asof_column) `ASOF JOIN`会从 `table_2` 中的用户事件时间戳找出和 `table_1` 中用户事件时间戳中最近的一个时间戳，来满足最接近匹配的条件。如果有得话，则相等的时间戳值是最接近的值。在此例中，`user_id` 列可用于条件匹配，`ev_time` 列可用于最接近匹配。在此例中，`event_1_1` 可以 JOIN `event_2_1`，`event_1_2` 可以JOIN `event_2_3`，但是 `event_2_2` 不能被JOIN。 -!!! note "注" - `ASOF JOIN`在 [JOIN](../../../engines/table-engines/special/join.md) 表引擎中 **不受** 支持。 +:::note +`ASOF JOIN`在 [JOIN](../../../engines/table-engines/special/join.md) 表引擎中 **不受** 支持。 +::: ## 分布式联接 {#global-join} diff --git a/docs/zh/sql-reference/statements/select/limit-by.md b/docs/zh/sql-reference/statements/select/limit-by.md index 50e3505b7fb..68b88bf8d7a 100644 --- a/docs/zh/sql-reference/statements/select/limit-by.md +++ b/docs/zh/sql-reference/statements/select/limit-by.md @@ -14,8 +14,9 @@ ClickHouse支持以下语法变体: 处理查询时，ClickHouse首先选择经由排序键排序过后的数据。排序键可以显式地使用[ORDER BY](order-by.md#select-order-by)从句指定，或隐式地使用表引擎使用的排序键（数据的顺序仅在使用[ORDER BY](order-by.md#select-order-by)时才可以保证，否则由于多线程处理，数据顺序会随机化）。然后ClickHouse执行`LIMIT n BY expressions`从句，将每一行按 `expressions` 的值进行分组，并对每一分组返回前`n`行。如果指定了`OFFSET`，那么对于每一分组，ClickHouse会跳过前`offset_value`行，接着返回前`n`行。如果`offset_value`大于某一分组的行数，ClickHouse会从分组返回0行。 -!!! note "注" - `LIMIT BY`与[LIMIT](../../../sql-reference/statements/select/limit.md)没有关系。它们可以在同一个查询中使用。 +:::note +`LIMIT BY`与[LIMIT](../../../sql-reference/statements/select/limit.md)没有关系。它们可以在同一个查询中使用。 +::: ## 例 {#examples} diff --git a/docs/zh/sql-reference/statements/select/sample.md b/docs/zh/sql-reference/statements/select/sample.md index f701bd3b805..0993958b029 100644 --- a/docs/zh/sql-reference/statements/select/sample.md +++ b/docs/zh/sql-reference/statements/select/sample.md @@ -15,8 +15,9 @@ sidebar_label: SAMPLE - 当您的原始数据不准确时，所以近似不会明显降低质量。 - 业务需求的目标是近似结果（为了成本效益，或者向高级用户推销确切结果）。 -!!! note "注" - 您只能使用采样中的表 [MergeTree](../../../engines/table-engines/mergetree-family/mergetree.md) 族，并且只有在表创建过程中指定了采样表达式（请参阅 [MergeTree引擎](../../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-creating-a-table)). +:::note +您只能使用采样中的表 [MergeTree](../../../engines/table-engines/mergetree-family/mergetree.md) 族，并且只有在表创建过程中指定了采样表达式（请参阅 [MergeTree引擎](../../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-creating-a-table)). +::: 下面列出了数据采样的功能: diff --git a/docs/zh/sql-reference/statements/select/where.md b/docs/zh/sql-reference/statements/select/where.md index fe165e8ad16..6c7183840c7 100644 --- a/docs/zh/sql-reference/statements/select/where.md +++ b/docs/zh/sql-reference/statements/select/where.md @@ -11,9 +11,10 @@ sidebar_label: WHERE 如果基础表引擎支持，`WHERE`表达式会使用索引和分区进行剪枝。 -!!! note "注" - 有一个叫做过滤优化 [prewhere](../../../sql-reference/statements/select/prewhere.md) 的东西. - +:::note +有一个叫做过滤优化 [prewhere](../../../sql-reference/statements/select/prewhere.md) 的东西. +::: + 如果需要测试一个 [NULL](../../../sql-reference/syntax.md#null-literal) 值，请使用 [IS NULL](../../operators/index.md#operator-is-null) and [IS NOT NULL](../../operators/index.md#is-not-null) 运算符或 [isNull](../../../sql-reference/functions/functions-for-nulls.md#isnull) 和 [isNotNull](../../../sql-reference/functions/functions-for-nulls.md#isnotnull) 函数。否则带有 NULL 的表达式永远不会通过。 **示例** diff --git a/docs/zh/sql-reference/statements/system.md b/docs/zh/sql-reference/statements/system.md index d8d60c28af5..1942d6fb79a 100644 --- a/docs/zh/sql-reference/statements/system.md +++ b/docs/zh/sql-reference/statements/system.md @@ -124,10 +124,9 @@ ClickHouse可以管理 [MergeTree](../../engines/table-engines/mergetree-family/ SYSTEM STOP MERGES [[db.]merge_tree_family_table_name] ``` - -!!! note "Note" - `DETACH / ATTACH` 表操作会在后台进行表的merge操作，甚至当所有MergeTree表的合并操作已经停止的情况下。 - +:::note +`DETACH / ATTACH` 表操作会在后台进行表的merge操作，甚至当所有MergeTree表的合并操作已经停止的情况下。 +::: ### START MERGES {#query_language-system-start-merges} diff --git a/docs/zh/sql-reference/table-functions/mysql.md b/docs/zh/sql-reference/table-functions/mysql.md index 6c9753b9b12..4efee2e616b 100644 --- a/docs/zh/sql-reference/table-functions/mysql.md +++ b/docs/zh/sql-reference/table-functions/mysql.md @@ -49,8 +49,9 @@ SELECT name FROM mysql(`mysql1:3306|mysql2:3306|mysql3:3306`, 'mysql_database', 与原始MySQL表具有相同列的表对象。 -!!! note "注意" - 在`INSERT`查询中为了区分`mysql(...)`与带有列名列表的表名的表函数，你必须使用关键字`FUNCTION`或`TABLE FUNCTION`。查看如下示例。 +:::note +在`INSERT`查询中为了区分`mysql(...)`与带有列名列表的表名的表函数，你必须使用关键字`FUNCTION`或`TABLE FUNCTION`。查看如下示例。 +::: ## 用法示例 {#usage-example} diff --git a/packages/clickhouse-keeper.service b/packages/clickhouse-keeper.service index 2809074c93a..e4ec5bf4ede 100644 --- a/packages/clickhouse-keeper.service +++ b/packages/clickhouse-keeper.service @@ -14,7 +14,8 @@ User=clickhouse Group=clickhouse Restart=always RestartSec=30 -RuntimeDirectory=%p # %p is resolved to the systemd unit name +# %p is resolved to the systemd unit name +RuntimeDirectory=%p ExecStart=/usr/bin/clickhouse-keeper --config=/etc/clickhouse-keeper/keeper_config.xml --pid-file=%t/%p/%p.pid # Minus means that this file is optional. EnvironmentFile=-/etc/default/%p diff --git a/programs/compressor/Compressor.cpp b/programs/compressor/Compressor.cpp index b60138b5692..cc25747702a 100644 --- a/programs/compressor/Compressor.cpp +++ b/programs/compressor/Compressor.cpp @@ -66,6 +66,7 @@ int mainEntryClickHouseCompressor(int argc, char ** argv) using namespace DB; namespace po = boost::program_options; + bool print_stacktrace = false; try { po::options_description desc = createOptionsDescription("Allowed options", getTerminalWidth()); @@ -84,6 +85,7 @@ int mainEntryClickHouseCompressor(int argc, char ** argv) ("level", po::value(), "compression level for codecs specified via flags") ("none", "use no compression instead of LZ4") ("stat", "print block statistics of compressed data") + ("stacktrace", "print stacktrace of exception") ; po::positional_options_description positional_desc; @@ -107,6 +109,7 @@ int mainEntryClickHouseCompressor(int argc, char ** argv) bool use_deflate_qpl = options.count("deflate_qpl"); bool stat_mode = options.count("stat"); bool use_none = options.count("none"); + print_stacktrace = options.count("stacktrace"); unsigned block_size = options["block-size"].as(); std::vector codecs; if (options.count("codec")) @@ -188,11 +191,12 @@ int mainEntryClickHouseCompressor(int argc, char ** argv) /// Compression CompressedWriteBuffer to(*wb, codec, block_size); copyData(*rb, to); + to.finalize(); } } catch (...) { - std::cerr << getCurrentExceptionMessage(true) << '\n'; + std::cerr << getCurrentExceptionMessage(print_stacktrace) << '\n'; return getCurrentExceptionCode(); } diff --git a/programs/copier/ClusterCopier.cpp b/programs/copier/ClusterCopier.cpp index bc882719a08..3a6261974d1 100644 --- a/programs/copier/ClusterCopier.cpp +++ b/programs/copier/ClusterCopier.cpp @@ -1757,8 +1757,7 @@ void ClusterCopier::dropParticularPartitionPieceFromAllHelpingTables(const TaskT LOG_INFO(log, "All helping tables dropped partition {}", partition_name); } -String ClusterCopier::getRemoteCreateTable( - const DatabaseAndTableName & table, Connection & connection, const Settings & settings) +String ClusterCopier::getRemoteCreateTable(const DatabaseAndTableName & table, Connection & connection, const Settings & settings) { auto remote_context = Context::createCopy(context); remote_context->setSettings(settings); @@ -1777,8 +1776,10 @@ ASTPtr ClusterCopier::getCreateTableForPullShard(const ConnectionTimeouts & time { /// Fetch and parse (possibly) new definition auto connection_entry = task_shard.info.pool->get(timeouts, &task_cluster->settings_pull, true); - String create_query_pull_str - = getRemoteCreateTable(task_shard.task_table.table_pull, *connection_entry, task_cluster->settings_pull); + String create_query_pull_str = getRemoteCreateTable( + task_shard.task_table.table_pull, + *connection_entry, + task_cluster->settings_pull); ParserCreateQuery parser_create_query; const auto & settings = getContext()->getSettingsRef(); @@ -1867,8 +1868,8 @@ std::set ClusterCopier::getShardPartitions(const ConnectionTimeouts & ti String query; { WriteBufferFromOwnString wb; - wb << "SELECT DISTINCT " << partition_name << " AS partition FROM" - << " " << getQuotedTable(task_shard.table_read_shard) << " ORDER BY partition DESC"; + wb << "SELECT " << partition_name << " AS partition FROM " + << getQuotedTable(task_shard.table_read_shard) << " GROUP BY partition ORDER BY partition DESC"; query = wb.str(); } @@ -2025,8 +2026,8 @@ UInt64 ClusterCopier::executeQueryOnCluster( /// For unknown reason global context is passed to IStorage::read() method /// So, task_identifier is passed as constructor argument. It is more obvious. auto remote_query_executor = std::make_shared( - *connections.back(), query, header, getContext(), - /*throttler=*/nullptr, Scalars(), Tables(), QueryProcessingStage::Complete); + *connections.back(), query, header, getContext(), + /*throttler=*/nullptr, Scalars(), Tables(), QueryProcessingStage::Complete); try { diff --git a/programs/install/Install.cpp b/programs/install/Install.cpp index d568012bb26..80f3b0bbc63 100644 --- a/programs/install/Install.cpp +++ b/programs/install/Install.cpp @@ -20,7 +20,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/programs/keeper/CMakeLists.txt b/programs/keeper/CMakeLists.txt index 9b01e6920a4..761335fb707 100644 --- a/programs/keeper/CMakeLists.txt +++ b/programs/keeper/CMakeLists.txt @@ -128,6 +128,7 @@ if (BUILD_STANDALONE_KEEPER) ch_contrib::lz4 ch_contrib::zstd ch_contrib::cityhash + ch_contrib::jemalloc common ch_contrib::double_conversion ch_contrib::dragonbox_to_chars pcg_random diff --git a/programs/odbc-bridge/ColumnInfoHandler.cpp b/programs/odbc-bridge/ColumnInfoHandler.cpp index 6e93246e59a..147ba43a51d 100644 --- a/programs/odbc-bridge/ColumnInfoHandler.cpp +++ b/programs/odbc-bridge/ColumnInfoHandler.cpp @@ -30,7 +30,7 @@ namespace DB namespace ErrorCodes { - extern const int LOGICAL_ERROR; + extern const int UNKNOWN_TABLE; extern const int BAD_ARGUMENTS; } @@ -180,8 +180,19 @@ void ODBCColumnsInfoHandler::handleRequest(HTTPServerRequest & request, HTTPServ columns.emplace_back(column_name, std::move(column_type)); } + /// Usually this should not happen, since in case of table does not + /// exists, the call should be succeeded. + /// However it is possible sometimes because internally there are two + /// queries in ClickHouse ODBC bridge: + /// - system.tables + /// - system.columns + /// And if between this two queries the table will be removed, them + /// there will be no columns + /// + /// Also sometimes system.columns can return empty result because of + /// the cached value of total tables to scan. if (columns.empty()) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Columns definition was not returned"); + throw Exception(ErrorCodes::UNKNOWN_TABLE, "Columns definition was not returned"); WriteBufferFromHTTPServerResponse out( response, diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 1486a51c710..711dfb3820a 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -41,6 +41,7 @@ #include #include #include +#include #include #include #include @@ -66,7 +67,6 @@ #include #include #include -#include #include #include #include @@ -773,6 +773,11 @@ try server_settings.max_io_thread_pool_free_size, server_settings.io_thread_pool_queue_size); + BackupsIOThreadPool::initialize( + server_settings.max_backups_io_thread_pool_size, + server_settings.max_backups_io_thread_pool_free_size, + server_settings.backups_io_thread_pool_queue_size); + /// Initialize global local cache for remote filesystem. if (config().has("local_cache_for_remote_fs")) { @@ -810,8 +815,6 @@ try } ); - ConnectionCollector::init(global_context, server_settings.max_threads_for_connection_collector); - bool has_zookeeper = config().has("zookeeper"); zkutil::ZooKeeperNodeCache main_config_zk_node_cache([&] { return global_context->getZooKeeper(); }); diff --git a/programs/server/config.xml b/programs/server/config.xml index 85cb299e188..0ea2de18e22 100644 --- a/programs/server/config.xml +++ b/programs/server/config.xml @@ -348,10 +348,6 @@ 16 --> - - - 0.9 diff --git a/src/Access/AccessBackup.cpp b/src/Access/AccessBackup.cpp index 53565e8e5d7..800a54e69b3 100644 --- a/src/Access/AccessBackup.cpp +++ b/src/Access/AccessBackup.cpp @@ -72,12 +72,11 @@ namespace return std::make_shared(buf.str()); } - static AccessEntitiesInBackup fromBackupEntry(const IBackupEntry & backup_entry, const String & file_path) + static AccessEntitiesInBackup fromBackupEntry(std::unique_ptr buf, const String & file_path) { try { AccessEntitiesInBackup res; - std::unique_ptr buf = backup_entry.getReadBuffer(); bool dependencies_found = false; @@ -343,8 +342,8 @@ void AccessRestorerFromBackup::addDataPath(const String & data_path) for (const String & filename : filenames) { String filepath_in_backup = data_path_in_backup_fs / filename; - auto backup_entry = backup->readFile(filepath_in_backup); - auto ab = AccessEntitiesInBackup::fromBackupEntry(*backup_entry, filepath_in_backup); + auto read_buffer_from_backup = backup->readFile(filepath_in_backup); + auto ab = AccessEntitiesInBackup::fromBackupEntry(std::move(read_buffer_from_backup), filepath_in_backup); boost::range::copy(ab.entities, std::back_inserter(entities)); boost::range::copy(ab.dependencies, std::inserter(dependencies, dependencies.end())); diff --git a/src/Access/Common/AccessType.h b/src/Access/Common/AccessType.h index f57cc2886e3..c73c0499fbe 100644 --- a/src/Access/Common/AccessType.h +++ b/src/Access/Common/AccessType.h @@ -15,6 +15,7 @@ enum class AccessType /// node_type either specifies access type's level (GLOBAL/DATABASE/TABLE/DICTIONARY/VIEW/COLUMNS), /// or specifies that the access type is a GROUP of other access types; /// parent_group_name is the name of the group containing this access type (or NONE if there is no such group). +/// NOTE A parent group must be declared AFTER all its children. #define APPLY_FOR_ACCESS_TYPES(M) \ M(SHOW_DATABASES, "", DATABASE, SHOW) /* allows to execute SHOW DATABASES, SHOW CREATE DATABASE, USE ; implicitly enabled by any grant on the database */\ @@ -86,8 +87,10 @@ enum class AccessType M(CREATE_VIEW, "", VIEW, CREATE) /* allows to execute {CREATE|ATTACH} VIEW; implicitly enabled by the grant CREATE_TABLE */\ M(CREATE_DICTIONARY, "", DICTIONARY, CREATE) /* allows to execute {CREATE|ATTACH} DICTIONARY */\ - M(CREATE_TEMPORARY_TABLE, "", GLOBAL, CREATE) /* allows to create and manipulate temporary tables; + M(CREATE_TEMPORARY_TABLE, "", GLOBAL, CREATE_ARBITRARY_TEMPORARY_TABLE) /* allows to create and manipulate temporary tables; implicitly enabled by the grant CREATE_TABLE on any table */ \ + M(CREATE_ARBITRARY_TEMPORARY_TABLE, "", GLOBAL, CREATE) /* allows to create and manipulate temporary tables + with arbitrary table engine */\ M(CREATE_FUNCTION, "", GLOBAL, CREATE) /* allows to execute CREATE FUNCTION */ \ M(CREATE_NAMED_COLLECTION, "", GLOBAL, CREATE) /* allows to execute CREATE NAMED COLLECTION */ \ M(CREATE, "", GROUP, ALL) /* allows to execute {CREATE|ATTACH} */ \ diff --git a/src/Access/ContextAccess.cpp b/src/Access/ContextAccess.cpp index fbaacb2263b..cc51183c51f 100644 --- a/src/Access/ContextAccess.cpp +++ b/src/Access/ContextAccess.cpp @@ -81,6 +81,11 @@ namespace if ((level == 0) && (max_flags_with_children & create_table)) res |= create_temporary_table; + /// CREATE TABLE (on any database/table) => CREATE_ARBITRARY_TEMPORARY_TABLE (global) + static const AccessFlags create_arbitrary_temporary_table = AccessType::CREATE_ARBITRARY_TEMPORARY_TABLE; + if ((level == 0) && (max_flags_with_children & create_table)) + res |= create_arbitrary_temporary_table; + /// ALTER_TTL => ALTER_MATERIALIZE_TTL static const AccessFlags alter_ttl = AccessType::ALTER_TTL; static const AccessFlags alter_materialize_ttl = AccessType::ALTER_MATERIALIZE_TTL; diff --git a/src/Access/EnabledRoles.h b/src/Access/EnabledRoles.h index e0d773db343..5de76abe409 100644 --- a/src/Access/EnabledRoles.h +++ b/src/Access/EnabledRoles.h @@ -44,10 +44,11 @@ private: friend class RoleCache; explicit EnabledRoles(const Params & params_); - void setRolesInfo(const std::shared_ptr & info_, scope_guard * notifications); - const Params params; + /// Called by RoleCache to store `EnabledRolesInfo` in this `EnabledRoles` after the calculation is done. + void setRolesInfo(const std::shared_ptr & info_, scope_guard * notifications); + std::shared_ptr info; mutable std::mutex info_mutex; diff --git a/src/Access/RoleCache.cpp b/src/Access/RoleCache.cpp index 308b771243e..bfc6200929d 100644 --- a/src/Access/RoleCache.cpp +++ b/src/Access/RoleCache.cpp @@ -57,7 +57,9 @@ namespace RoleCache::RoleCache(const AccessControl & access_control_) - : access_control(access_control_), cache(600000 /* 10 minutes */) {} + : access_control(access_control_), cache(600000 /* 10 minutes */) +{ +} RoleCache::~RoleCache() = default; @@ -70,18 +72,18 @@ RoleCache::getEnabledRoles(const std::vector & roles, const std::vectorsecond.lock(); - if (from_cache) - return from_cache; - enabled_roles.erase(it); + if (auto enabled_roles = it->second.enabled_roles.lock()) + return enabled_roles; + enabled_roles_by_params.erase(it); } auto res = std::shared_ptr(new EnabledRoles(params)); - collectEnabledRoles(*res, nullptr); - enabled_roles.emplace(std::move(params), res); + SubscriptionsOnRoles subscriptions_on_roles; + collectEnabledRoles(*res, subscriptions_on_roles, nullptr); + enabled_roles_by_params.emplace(std::move(params), EnabledRolesWithSubscriptions{res, std::move(subscriptions_on_roles)}); return res; } @@ -90,21 +92,23 @@ void RoleCache::collectEnabledRoles(scope_guard * notifications) { /// `mutex` is already locked. - for (auto i = enabled_roles.begin(), e = enabled_roles.end(); i != e;) + for (auto i = enabled_roles_by_params.begin(), e = enabled_roles_by_params.end(); i != e;) { - auto elem = i->second.lock(); - if (!elem) - i = enabled_roles.erase(i); + auto & item = i->second; + if (auto enabled_roles = item.enabled_roles.lock()) + { + collectEnabledRoles(*enabled_roles, item.subscriptions_on_roles, notifications); + ++i; + } else { - collectEnabledRoles(*elem, notifications); - ++i; + i = enabled_roles_by_params.erase(i); } } } -void RoleCache::collectEnabledRoles(EnabledRoles & enabled, scope_guard * notifications) +void RoleCache::collectEnabledRoles(EnabledRoles & enabled_roles, SubscriptionsOnRoles & subscriptions_on_roles, scope_guard * notifications) { /// `mutex` is already locked. @@ -112,43 +116,57 @@ void RoleCache::collectEnabledRoles(EnabledRoles & enabled, scope_guard * notifi auto new_info = std::make_shared(); boost::container::flat_set skip_ids; - auto get_role_function = [this](const UUID & id) { return getRole(id); }; + /// We need to collect and keep not only enabled roles but also subscriptions for them to be able to recalculate EnabledRolesInfo when some of the roles change. + SubscriptionsOnRoles new_subscriptions_on_roles; + new_subscriptions_on_roles.reserve(subscriptions_on_roles.size()); - for (const auto & current_role : enabled.params.current_roles) + auto get_role_function = [this, &subscriptions_on_roles](const UUID & id) TSA_NO_THREAD_SAFETY_ANALYSIS { return getRole(id, subscriptions_on_roles); }; + + for (const auto & current_role : enabled_roles.params.current_roles) collectRoles(*new_info, skip_ids, get_role_function, current_role, true, false); - for (const auto & current_role : enabled.params.current_roles_with_admin_option) + for (const auto & current_role : enabled_roles.params.current_roles_with_admin_option) collectRoles(*new_info, skip_ids, get_role_function, current_role, true, true); + /// Remove duplicates from `subscriptions_on_roles`. + std::sort(new_subscriptions_on_roles.begin(), new_subscriptions_on_roles.end()); + new_subscriptions_on_roles.erase(std::unique(new_subscriptions_on_roles.begin(), new_subscriptions_on_roles.end()), new_subscriptions_on_roles.end()); + subscriptions_on_roles = std::move(new_subscriptions_on_roles); + /// Collect data from the collected roles. - enabled.setRolesInfo(new_info, notifications); + enabled_roles.setRolesInfo(new_info, notifications); } -RolePtr RoleCache::getRole(const UUID & role_id) +RolePtr RoleCache::getRole(const UUID & role_id, SubscriptionsOnRoles & subscriptions_on_roles) { /// `mutex` is already locked. auto role_from_cache = cache.get(role_id); if (role_from_cache) + { + subscriptions_on_roles.emplace_back(role_from_cache->second); return role_from_cache->first; + } - auto subscription = access_control.subscribeForChanges(role_id, - [this, role_id](const UUID &, const AccessEntityPtr & entity) + auto on_role_changed_or_removed = [this, role_id](const UUID &, const AccessEntityPtr & entity) { auto changed_role = entity ? typeid_cast(entity) : nullptr; if (changed_role) roleChanged(role_id, changed_role); else roleRemoved(role_id); - }); + }; + + auto subscription_on_role = std::make_shared(access_control.subscribeForChanges(role_id, on_role_changed_or_removed)); auto role = access_control.tryRead(role_id); if (role) { - auto cache_value = Poco::SharedPtr>( - new std::pair{role, std::move(subscription)}); + auto cache_value = Poco::SharedPtr>>( + new std::pair>{role, subscription_on_role}); cache.add(role_id, cache_value); + subscriptions_on_roles.emplace_back(subscription_on_role); return role; } @@ -162,12 +180,17 @@ void RoleCache::roleChanged(const UUID & role_id, const RolePtr & changed_role) scope_guard notifications; std::lock_guard lock{mutex}; + auto role_from_cache = cache.get(role_id); - if (!role_from_cache) - return; - role_from_cache->first = changed_role; - cache.update(role_id, role_from_cache); - collectEnabledRoles(¬ifications); + if (role_from_cache) + { + /// We update the role stored in a cache entry only if that entry has not expired yet. + role_from_cache->first = changed_role; + cache.update(role_id, role_from_cache); + } + + /// An enabled role for some users has been changed, we need to recalculate the access rights. + collectEnabledRoles(¬ifications); /// collectEnabledRoles() must be called with the `mutex` locked. } @@ -177,8 +200,12 @@ void RoleCache::roleRemoved(const UUID & role_id) scope_guard notifications; std::lock_guard lock{mutex}; + + /// If a cache entry with the role has expired already, that remove() will do nothing. cache.remove(role_id); - collectEnabledRoles(¬ifications); + + /// An enabled role for some users has been removed, we need to recalculate the access rights. + collectEnabledRoles(¬ifications); /// collectEnabledRoles() must be called with the `mutex` locked. } } diff --git a/src/Access/RoleCache.h b/src/Access/RoleCache.h index 51c415d4d1d..24f19cb9d94 100644 --- a/src/Access/RoleCache.h +++ b/src/Access/RoleCache.h @@ -24,15 +24,29 @@ public: const std::vector & current_roles_with_admin_option); private: - void collectEnabledRoles(scope_guard * notifications); - void collectEnabledRoles(EnabledRoles & enabled, scope_guard * notifications); - RolePtr getRole(const UUID & role_id); + using SubscriptionsOnRoles = std::vector>; + + void collectEnabledRoles(scope_guard * notifications) TSA_REQUIRES(mutex); + void collectEnabledRoles(EnabledRoles & enabled_roles, SubscriptionsOnRoles & subscriptions_on_roles, scope_guard * notifications) TSA_REQUIRES(mutex); + RolePtr getRole(const UUID & role_id, SubscriptionsOnRoles & subscriptions_on_roles) TSA_REQUIRES(mutex); void roleChanged(const UUID & role_id, const RolePtr & changed_role); void roleRemoved(const UUID & role_id); const AccessControl & access_control; - Poco::AccessExpireCache> cache; - std::map> enabled_roles; + + Poco::AccessExpireCache>> TSA_GUARDED_BY(mutex) cache; + + struct EnabledRolesWithSubscriptions + { + std::weak_ptr enabled_roles; + + /// We need to keep subscriptions for all enabled roles to be able to recalculate EnabledRolesInfo when some of the roles change. + /// `cache` also keeps subscriptions but that's not enough because values can be purged from the `cache` anytime. + SubscriptionsOnRoles subscriptions_on_roles; + }; + + std::map TSA_GUARDED_BY(mutex) enabled_roles_by_params; + mutable std::mutex mutex; }; diff --git a/src/Analyzer/ColumnNode.h b/src/Analyzer/ColumnNode.h index e378bc5f3d0..79c0e23c86f 100644 --- a/src/Analyzer/ColumnNode.h +++ b/src/Analyzer/ColumnNode.h @@ -3,6 +3,7 @@ #include #include +#include namespace DB { @@ -117,6 +118,11 @@ public: return column.type; } + void convertToNullable() override + { + column.type = makeNullableSafe(column.type); + } + void dumpTreeImpl(WriteBuffer & buffer, FormatState & state, size_t indent) const override; protected: diff --git a/src/Analyzer/FunctionNode.cpp b/src/Analyzer/FunctionNode.cpp index 7961bfbae31..fe170c8482e 100644 --- a/src/Analyzer/FunctionNode.cpp +++ b/src/Analyzer/FunctionNode.cpp @@ -2,18 +2,21 @@ #include #include -#include -#include #include #include +#include +#include + #include #include #include +#include +#include #include namespace DB @@ -44,17 +47,29 @@ const DataTypes & FunctionNode::getArgumentTypes() const ColumnsWithTypeAndName FunctionNode::getArgumentColumns() const { const auto & arguments = getArguments().getNodes(); + size_t arguments_size = arguments.size(); + ColumnsWithTypeAndName argument_columns; argument_columns.reserve(arguments.size()); - for (const auto & arg : arguments) + for (size_t i = 0; i < arguments_size; ++i) { - ColumnWithTypeAndName argument; - argument.type = arg->getResultType(); - if (auto * constant = arg->as()) - argument.column = argument.type->createColumnConst(1, constant->getValue()); - argument_columns.push_back(std::move(argument)); + const auto & argument = arguments[i]; + + ColumnWithTypeAndName argument_column; + + if (isNameOfInFunction(function_name) && i == 1) + argument_column.type = std::make_shared(); + else + argument_column.type = argument->getResultType(); + + auto * constant = argument->as(); + if (constant && !isNotCreatable(argument_column.type)) + argument_column.column = argument_column.type->createColumnConst(1, constant->getValue()); + + argument_columns.push_back(std::move(argument_column)); } + return argument_columns; } @@ -99,7 +114,7 @@ void FunctionNode::dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state buffer << ", function_type: " << function_type; if (function) - buffer << ", result_type: " + function->getResultType()->getName(); + buffer << ", result_type: " + getResultType()->getName(); const auto & parameters = getParameters(); if (!parameters.getNodes().empty()) @@ -177,6 +192,7 @@ QueryTreeNodePtr FunctionNode::cloneImpl() const */ result_function->function = function; result_function->kind = kind; + result_function->wrap_with_nullable = wrap_with_nullable; return result_function; } diff --git a/src/Analyzer/FunctionNode.h b/src/Analyzer/FunctionNode.h index 6819232b4be..89a684c1d0f 100644 --- a/src/Analyzer/FunctionNode.h +++ b/src/Analyzer/FunctionNode.h @@ -8,6 +8,7 @@ #include #include #include +#include #include namespace DB @@ -187,7 +188,16 @@ public: throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "Function node with name '{}' is not resolved", function_name); - return function->getResultType(); + auto type = function->getResultType(); + if (wrap_with_nullable) + return makeNullableSafe(type); + return type; + } + + void convertToNullable() override + { + chassert(kind == FunctionKind::ORDINARY); + wrap_with_nullable = true; } void dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state, size_t indent) const override; @@ -205,6 +215,7 @@ private: String function_name; FunctionKind kind = FunctionKind::UNKNOWN; IResolvedFunctionPtr function; + bool wrap_with_nullable = false; static constexpr size_t parameters_child_index = 0; static constexpr size_t arguments_child_index = 1; diff --git a/src/Analyzer/IQueryTreeNode.h b/src/Analyzer/IQueryTreeNode.h index 157bbd1b951..e344dd66fbc 100644 --- a/src/Analyzer/IQueryTreeNode.h +++ b/src/Analyzer/IQueryTreeNode.h @@ -90,6 +90,11 @@ public: throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "Method getResultType is not supported for {} query node", getNodeTypeName()); } + virtual void convertToNullable() + { + throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "Method convertToNullable is not supported for {} query node", getNodeTypeName()); + } + struct CompareOptions { bool compare_aliases = true; diff --git a/src/Analyzer/InDepthQueryTreeVisitor.h b/src/Analyzer/InDepthQueryTreeVisitor.h index af69fc55589..1cc48fb1e53 100644 --- a/src/Analyzer/InDepthQueryTreeVisitor.h +++ b/src/Analyzer/InDepthQueryTreeVisitor.h @@ -99,8 +99,9 @@ class InDepthQueryTreeVisitorWithContext public: using VisitQueryTreeNodeType = std::conditional_t; - explicit InDepthQueryTreeVisitorWithContext(ContextPtr context) + explicit InDepthQueryTreeVisitorWithContext(ContextPtr context, size_t initial_subquery_depth = 0) : current_context(std::move(context)) + , subquery_depth(initial_subquery_depth) {} /// Return true if visitor should traverse tree top to bottom, false otherwise @@ -125,11 +126,17 @@ public: return current_context->getSettingsRef(); } + size_t getSubqueryDepth() const + { + return subquery_depth; + } + void visit(VisitQueryTreeNodeType & query_tree_node) { auto current_scope_context_ptr = current_context; SCOPE_EXIT( current_context = std::move(current_scope_context_ptr); + --subquery_depth; ); if (auto * query_node = query_tree_node->template as()) @@ -137,6 +144,8 @@ public: else if (auto * union_node = query_tree_node->template as()) current_context = union_node->getContext(); + ++subquery_depth; + bool traverse_top_to_bottom = getDerived().shouldTraverseTopToBottom(); if (!traverse_top_to_bottom) visitChildren(query_tree_node); @@ -145,7 +154,12 @@ public: if (traverse_top_to_bottom) visitChildren(query_tree_node); + + getDerived().leaveImpl(query_tree_node); } + + void leaveImpl(VisitQueryTreeNodeType & node [[maybe_unused]]) + {} private: Derived & getDerived() { @@ -172,6 +186,7 @@ private: } ContextPtr current_context; + size_t subquery_depth = 0; }; template diff --git a/src/Analyzer/JoinNode.h b/src/Analyzer/JoinNode.h index 0d856985794..f58fe3f1af5 100644 --- a/src/Analyzer/JoinNode.h +++ b/src/Analyzer/JoinNode.h @@ -106,6 +106,12 @@ public: return locality; } + /// Set join locality + void setLocality(JoinLocality locality_value) + { + locality = locality_value; + } + /// Get join strictness JoinStrictness getStrictness() const { diff --git a/src/Analyzer/Passes/ArrayExistsToHasPass.cpp b/src/Analyzer/Passes/ArrayExistsToHasPass.cpp index b4b8b5b4579..c0f958588f1 100644 --- a/src/Analyzer/Passes/ArrayExistsToHasPass.cpp +++ b/src/Analyzer/Passes/ArrayExistsToHasPass.cpp @@ -1,3 +1,5 @@ +#include + #include #include @@ -8,71 +10,85 @@ #include #include -#include "ArrayExistsToHasPass.h" - namespace DB { + namespace { - class RewriteArrayExistsToHasVisitor : public InDepthQueryTreeVisitorWithContext + +class RewriteArrayExistsToHasVisitor : public InDepthQueryTreeVisitorWithContext +{ +public: + using Base = InDepthQueryTreeVisitorWithContext; + using Base::Base; + + void visitImpl(QueryTreeNodePtr & node) { - public: - using Base = InDepthQueryTreeVisitorWithContext; - using Base::Base; + if (!getSettings().optimize_rewrite_array_exists_to_has) + return; - void visitImpl(QueryTreeNodePtr & node) + auto * array_exists_function_node = node->as(); + if (!array_exists_function_node || array_exists_function_node->getFunctionName() != "arrayExists") + return; + + auto & array_exists_function_arguments_nodes = array_exists_function_node->getArguments().getNodes(); + if (array_exists_function_arguments_nodes.size() != 2) + return; + + /// lambda function must be like: x -> x = elem + auto * lambda_node = array_exists_function_arguments_nodes[0]->as(); + if (!lambda_node) + return; + + auto & lambda_arguments_nodes = lambda_node->getArguments().getNodes(); + if (lambda_arguments_nodes.size() != 1) + return; + + const auto & lambda_argument_column_node = lambda_arguments_nodes[0]; + if (lambda_argument_column_node->getNodeType() != QueryTreeNodeType::COLUMN) + return; + + auto * filter_node = lambda_node->getExpression()->as(); + if (!filter_node || filter_node->getFunctionName() != "equals") + return; + + const auto & filter_arguments_nodes = filter_node->getArguments().getNodes(); + if (filter_arguments_nodes.size() != 2) + return; + + const auto & filter_lhs_argument_node = filter_arguments_nodes[0]; + auto filter_lhs_argument_node_type = filter_lhs_argument_node->getNodeType(); + + const auto & filter_rhs_argument_node = filter_arguments_nodes[1]; + auto filter_rhs_argument_node_type = filter_rhs_argument_node->getNodeType(); + + QueryTreeNodePtr has_constant_element_argument; + + if (filter_lhs_argument_node_type == QueryTreeNodeType::COLUMN && + filter_rhs_argument_node_type == QueryTreeNodeType::CONSTANT && + filter_lhs_argument_node->isEqual(*lambda_argument_column_node)) { - if (!getSettings().optimize_rewrite_array_exists_to_has) - return; - - auto * function_node = node->as(); - if (!function_node || function_node->getFunctionName() != "arrayExists") - return; - - auto & function_arguments_nodes = function_node->getArguments().getNodes(); - if (function_arguments_nodes.size() != 2) - return; - - /// lambda function must be like: x -> x = elem - auto * lambda_node = function_arguments_nodes[0]->as(); - if (!lambda_node) - return; - - auto & lambda_arguments_nodes = lambda_node->getArguments().getNodes(); - if (lambda_arguments_nodes.size() != 1) - return; - auto * column_node = lambda_arguments_nodes[0]->as(); - - auto * filter_node = lambda_node->getExpression()->as(); - if (!filter_node || filter_node->getFunctionName() != "equals") - return; - - auto filter_arguments_nodes = filter_node->getArguments().getNodes(); - if (filter_arguments_nodes.size() != 2) - return; - - ColumnNode * filter_column_node = nullptr; - if (filter_arguments_nodes[1]->as() && (filter_column_node = filter_arguments_nodes[0]->as()) - && filter_column_node->getColumnName() == column_node->getColumnName()) - { - /// Rewrite arrayExists(x -> x = elem, arr) -> has(arr, elem) - function_arguments_nodes[0] = std::move(function_arguments_nodes[1]); - function_arguments_nodes[1] = std::move(filter_arguments_nodes[1]); - function_node->resolveAsFunction( - FunctionFactory::instance().get("has", getContext())->build(function_node->getArgumentColumns())); - } - else if ( - filter_arguments_nodes[0]->as() && (filter_column_node = filter_arguments_nodes[1]->as()) - && filter_column_node->getColumnName() == column_node->getColumnName()) - { - /// Rewrite arrayExists(x -> elem = x, arr) -> has(arr, elem) - function_arguments_nodes[0] = std::move(function_arguments_nodes[1]); - function_arguments_nodes[1] = std::move(filter_arguments_nodes[0]); - function_node->resolveAsFunction( - FunctionFactory::instance().get("has", getContext())->build(function_node->getArgumentColumns())); - } + /// Rewrite arrayExists(x -> x = elem, arr) -> has(arr, elem) + has_constant_element_argument = filter_rhs_argument_node; } - }; + else if (filter_lhs_argument_node_type == QueryTreeNodeType::CONSTANT && + filter_rhs_argument_node_type == QueryTreeNodeType::COLUMN && + filter_rhs_argument_node->isEqual(*lambda_argument_column_node)) + { + /// Rewrite arrayExists(x -> elem = x, arr) -> has(arr, elem) + has_constant_element_argument = filter_lhs_argument_node; + } + else + { + return; + } + + auto has_function = FunctionFactory::instance().get("has", getContext()); + array_exists_function_arguments_nodes[0] = std::move(array_exists_function_arguments_nodes[1]); + array_exists_function_arguments_nodes[1] = std::move(has_constant_element_argument); + array_exists_function_node->resolveAsFunction(has_function->build(array_exists_function_node->getArgumentColumns())); + } +}; } diff --git a/src/Analyzer/Passes/ArrayExistsToHasPass.h b/src/Analyzer/Passes/ArrayExistsToHasPass.h index 7d9d1cf3d68..8f4623116e3 100644 --- a/src/Analyzer/Passes/ArrayExistsToHasPass.h +++ b/src/Analyzer/Passes/ArrayExistsToHasPass.h @@ -4,8 +4,15 @@ namespace DB { -/// Rewrite possible 'arrayExists(func, arr)' to 'has(arr, elem)' to improve performance -/// arrayExists(x -> x = 1, arr) -> has(arr, 1) + +/** Rewrite possible 'arrayExists(func, arr)' to 'has(arr, elem)' to improve performance. + * + * Example: SELECT arrayExists(x -> x = 1, arr); + * Result: SELECT has(arr, 1); + * + * Example: SELECT arrayExists(x -> 1 = x, arr); + * Result: SELECT has(arr, 1); + */ class RewriteArrayExistsToHasPass final : public IQueryTreePass { public: @@ -15,4 +22,5 @@ public: void run(QueryTreeNodePtr query_tree_node, ContextPtr context) override; }; + } diff --git a/src/Analyzer/Passes/AutoFinalOnQueryPass.cpp b/src/Analyzer/Passes/AutoFinalOnQueryPass.cpp index fdf818681d7..15326ca1dc8 100644 --- a/src/Analyzer/Passes/AutoFinalOnQueryPass.cpp +++ b/src/Analyzer/Passes/AutoFinalOnQueryPass.cpp @@ -22,8 +22,7 @@ public: void visitImpl(QueryTreeNodePtr & node) { - const auto & context = getContext(); - if (!context->getSettingsRef().final) + if (!getSettings().final) return; const auto * query_node = node->as(); @@ -43,7 +42,7 @@ private: return; const auto & storage = table_node ? table_node->getStorage() : table_function_node->getStorage(); - bool is_final_supported = storage && storage->supportsFinal() && !storage->isRemote(); + bool is_final_supported = storage && storage->supportsFinal(); if (!is_final_supported) return; diff --git a/src/Analyzer/Passes/GroupingFunctionsResolvePass.cpp b/src/Analyzer/Passes/GroupingFunctionsResolvePass.cpp index 20f1713f8c2..0cf5310a3ad 100644 --- a/src/Analyzer/Passes/GroupingFunctionsResolvePass.cpp +++ b/src/Analyzer/Passes/GroupingFunctionsResolvePass.cpp @@ -32,17 +32,17 @@ enum class GroupByKind GROUPING_SETS }; -class GroupingFunctionResolveVisitor : public InDepthQueryTreeVisitor +class GroupingFunctionResolveVisitor : public InDepthQueryTreeVisitorWithContext { public: GroupingFunctionResolveVisitor(GroupByKind group_by_kind_, QueryTreeNodePtrWithHashMap aggregation_key_to_index_, ColumnNumbersList grouping_sets_keys_indices_, ContextPtr context_) - : group_by_kind(group_by_kind_) + : InDepthQueryTreeVisitorWithContext(std::move(context_)) + , group_by_kind(group_by_kind_) , aggregation_key_to_index(std::move(aggregation_key_to_index_)) , grouping_sets_keys_indexes(std::move(grouping_sets_keys_indices_)) - , context(std::move(context_)) { } @@ -71,7 +71,7 @@ public: FunctionOverloadResolverPtr grouping_function_resolver; bool add_grouping_set_column = false; - bool force_grouping_standard_compatibility = context->getSettingsRef().force_grouping_standard_compatibility; + bool force_grouping_standard_compatibility = getSettings().force_grouping_standard_compatibility; size_t aggregation_keys_size = aggregation_key_to_index.size(); switch (group_by_kind) @@ -132,7 +132,6 @@ private: GroupByKind group_by_kind; QueryTreeNodePtrWithHashMap aggregation_key_to_index; ColumnNumbersList grouping_sets_keys_indexes; - ContextPtr context; }; void resolveGroupingFunctions(QueryTreeNodePtr & query_node, ContextPtr context) @@ -164,12 +163,17 @@ void resolveGroupingFunctions(QueryTreeNodePtr & query_node, ContextPtr context) grouping_sets_used_aggregation_keys_list.emplace_back(); auto & grouping_sets_used_aggregation_keys = grouping_sets_used_aggregation_keys_list.back(); + QueryTreeNodePtrWithHashSet used_keys_in_set; + for (auto & grouping_set_key_node : grouping_set_keys_list_node_typed.getNodes()) { + if (used_keys_in_set.contains(grouping_set_key_node)) + continue; + used_keys_in_set.insert(grouping_set_key_node); + grouping_sets_used_aggregation_keys.push_back(grouping_set_key_node); + if (aggregation_key_to_index.contains(grouping_set_key_node)) continue; - - grouping_sets_used_aggregation_keys.push_back(grouping_set_key_node); aggregation_key_to_index.emplace(grouping_set_key_node, aggregation_node_index); ++aggregation_node_index; } diff --git a/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp b/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp new file mode 100644 index 00000000000..3d65035f9fd --- /dev/null +++ b/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp @@ -0,0 +1,241 @@ +#include + +#include + +#include +#include +#include +#include + +namespace DB +{ + +class LogicalExpressionOptimizerVisitor : public InDepthQueryTreeVisitorWithContext +{ +public: + using Base = InDepthQueryTreeVisitorWithContext; + + explicit LogicalExpressionOptimizerVisitor(ContextPtr context) + : Base(std::move(context)) + {} + + void visitImpl(QueryTreeNodePtr & node) + { + auto * function_node = node->as(); + + if (!function_node) + return; + + if (function_node->getFunctionName() == "or") + { + tryReplaceOrEqualsChainWithIn(node); + return; + } + + if (function_node->getFunctionName() == "and") + { + tryReplaceAndEqualsChainsWithConstant(node); + return; + } + } +private: + void tryReplaceAndEqualsChainsWithConstant(QueryTreeNodePtr & node) + { + auto & function_node = node->as(); + assert(function_node.getFunctionName() == "and"); + + if (function_node.getResultType()->isNullable()) + return; + + QueryTreeNodes and_operands; + + QueryTreeNodePtrWithHashMap node_to_constants; + + for (const auto & argument : function_node.getArguments()) + { + auto * argument_function = argument->as(); + if (!argument_function || argument_function->getFunctionName() != "equals") + { + and_operands.push_back(argument); + continue; + } + + const auto & equals_arguments = argument_function->getArguments().getNodes(); + const auto & lhs = equals_arguments[0]; + const auto & rhs = equals_arguments[1]; + + const auto has_and_with_different_constant = [&](const QueryTreeNodePtr & expression, const ConstantNode * constant) + { + if (auto it = node_to_constants.find(expression); it != node_to_constants.end()) + { + if (!it->second->isEqual(*constant)) + return true; + } + else + { + node_to_constants.emplace(expression, constant); + and_operands.push_back(argument); + } + + return false; + }; + + bool collapse_to_false = false; + + if (const auto * lhs_literal = lhs->as()) + collapse_to_false = has_and_with_different_constant(rhs, lhs_literal); + else if (const auto * rhs_literal = rhs->as()) + collapse_to_false = has_and_with_different_constant(lhs, rhs_literal); + else + and_operands.push_back(argument); + + if (collapse_to_false) + { + auto false_value = std::make_shared(0u, function_node.getResultType()); + auto false_node = std::make_shared(std::move(false_value)); + node = std::move(false_node); + return; + } + } + + if (and_operands.size() == function_node.getArguments().getNodes().size()) + return; + + if (and_operands.size() == 1) + { + /// AND operator can have UInt8 or bool as its type. + /// bool is used if a bool constant is at least one operand. + /// Because we reduce the number of operands here by eliminating the same equality checks, + /// the only situation we can end up here is we had AND check where all the equality checks are the same so we know the type is UInt8. + /// Otherwise, we will have > 1 operands and we don't have to do anything. + assert(!function_node.getResultType()->isNullable() && and_operands[0]->getResultType()->equals(*function_node.getResultType())); + node = std::move(and_operands[0]); + return; + } + + auto and_function_resolver = FunctionFactory::instance().get("and", getContext()); + function_node.getArguments().getNodes() = std::move(and_operands); + function_node.resolveAsFunction(and_function_resolver); + } + + void tryReplaceOrEqualsChainWithIn(QueryTreeNodePtr & node) + { + auto & function_node = node->as(); + assert(function_node.getFunctionName() == "or"); + + QueryTreeNodes or_operands; + + QueryTreeNodePtrWithHashMap node_to_equals_functions; + QueryTreeNodePtrWithHashMap node_to_constants; + + for (const auto & argument : function_node.getArguments()) + { + auto * argument_function = argument->as(); + if (!argument_function || argument_function->getFunctionName() != "equals") + { + or_operands.push_back(argument); + continue; + } + + /// collect all equality checks (x = value) + + const auto & equals_arguments = argument_function->getArguments().getNodes(); + const auto & lhs = equals_arguments[0]; + const auto & rhs = equals_arguments[1]; + + const auto add_equals_function_if_not_present = [&](const auto & expression_node, const ConstantNode * constant) + { + auto & constant_set = node_to_constants[expression_node]; + if (!constant_set.contains(constant)) + { + constant_set.insert(constant); + node_to_equals_functions[expression_node].push_back(argument); + } + }; + + if (const auto * lhs_literal = lhs->as()) + add_equals_function_if_not_present(rhs, lhs_literal); + else if (const auto * rhs_literal = rhs->as()) + add_equals_function_if_not_present(lhs, rhs_literal); + else + or_operands.push_back(argument); + } + + auto in_function_resolver = FunctionFactory::instance().get("in", getContext()); + + for (auto & [expression, equals_functions] : node_to_equals_functions) + { + const auto & settings = getSettings(); + if (equals_functions.size() < settings.optimize_min_equality_disjunction_chain_length && !expression.node->getResultType()->lowCardinality()) + { + std::move(equals_functions.begin(), equals_functions.end(), std::back_inserter(or_operands)); + continue; + } + + Tuple args; + args.reserve(equals_functions.size()); + /// first we create tuple from RHS of equals functions + for (const auto & equals : equals_functions) + { + const auto * equals_function = equals->as(); + assert(equals_function && equals_function->getFunctionName() == "equals"); + + const auto & equals_arguments = equals_function->getArguments().getNodes(); + if (const auto * rhs_literal = equals_arguments[1]->as()) + { + args.push_back(rhs_literal->getValue()); + } + else + { + const auto * lhs_literal = equals_arguments[0]->as(); + assert(lhs_literal); + args.push_back(lhs_literal->getValue()); + } + } + + auto rhs_node = std::make_shared(std::move(args)); + + auto in_function = std::make_shared("in"); + + QueryTreeNodes in_arguments; + in_arguments.reserve(2); + in_arguments.push_back(expression.node); + in_arguments.push_back(std::move(rhs_node)); + + in_function->getArguments().getNodes() = std::move(in_arguments); + in_function->resolveAsFunction(in_function_resolver); + + or_operands.push_back(std::move(in_function)); + } + + if (or_operands.size() == function_node.getArguments().getNodes().size()) + return; + + if (or_operands.size() == 1) + { + /// if the result type of operand is the same as the result type of OR + /// we can replace OR with the operand + if (or_operands[0]->getResultType()->equals(*function_node.getResultType())) + { + assert(!function_node.getResultType()->isNullable()); + node = std::move(or_operands[0]); + return; + } + + /// otherwise add a stub 0 to make OR correct + or_operands.push_back(std::make_shared(static_cast(0))); + } + + auto or_function_resolver = FunctionFactory::instance().get("or", getContext()); + function_node.getArguments().getNodes() = std::move(or_operands); + function_node.resolveAsFunction(or_function_resolver); + } +}; + +void LogicalExpressionOptimizerPass::run(QueryTreeNodePtr query_tree_node, ContextPtr context) +{ + LogicalExpressionOptimizerVisitor visitor(std::move(context)); + visitor.visit(query_tree_node); +} + +} diff --git a/src/Analyzer/Passes/LogicalExpressionOptimizerPass.h b/src/Analyzer/Passes/LogicalExpressionOptimizerPass.h new file mode 100644 index 00000000000..05c10ddc685 --- /dev/null +++ b/src/Analyzer/Passes/LogicalExpressionOptimizerPass.h @@ -0,0 +1,82 @@ +#pragma once + +#include + +namespace DB +{ + +/** + * This pass tries to do optimizations on logical expression: + * + * 1. Replaces chains of equality functions inside an OR with a single IN operator. + * The replacement is done if: + * - one of the operands of the equality function is a constant + * - length of chain is at least 'optimize_min_equality_disjunction_chain_length' long OR the expression has type of LowCardinality + * + * E.g. (optimize_min_equality_disjunction_chain_length = 2) + * ------------------------------- + * SELECT * + * FROM table + * WHERE a = 1 OR b = 'test' OR a = 2; + * + * will be transformed into + * + * SELECT * + * FROM TABLE + * WHERE b = 'test' OR a IN (1, 2); + * ------------------------------- + * + * 2. Removes duplicate OR checks + * ------------------------------- + * SELECT * + * FROM table + * WHERE a = 1 OR b = 'test' OR a = 1; + * + * will be transformed into + * + * SELECT * + * FROM TABLE + * WHERE a = 1 OR b = 'test'; + * ------------------------------- + * + * 3. Replaces AND chains with a single constant. + * The replacement is done if: + * - one of the operands of the equality function is a constant + * - constants are different for same expression + * ------------------------------- + * SELECT * + * FROM table + * WHERE a = 1 AND b = 'test' AND a = 2; + * + * will be transformed into + * + * SELECT * + * FROM TABLE + * WHERE 0; + * ------------------------------- + * + * 4. Removes duplicate AND checks + * ------------------------------- + * SELECT * + * FROM table + * WHERE a = 1 AND b = 'test' AND a = 1; + * + * will be transformed into + * + * SELECT * + * FROM TABLE + * WHERE a = 1 AND b = 'test'; + * ------------------------------- + */ + +class LogicalExpressionOptimizerPass final : public IQueryTreePass +{ +public: + String getName() override { return "LogicalExpressionOptimizer"; } + + String getDescription() override { return "Transform equality chain to a single IN function or a constant if possible"; } + + void run(QueryTreeNodePtr query_tree_node, ContextPtr context) override; +}; + +} diff --git a/src/Analyzer/Passes/OptimizeGroupByFunctionKeysPass.cpp b/src/Analyzer/Passes/OptimizeGroupByFunctionKeysPass.cpp index f6c4d2bc15d..c97645219da 100644 --- a/src/Analyzer/Passes/OptimizeGroupByFunctionKeysPass.cpp +++ b/src/Analyzer/Passes/OptimizeGroupByFunctionKeysPass.cpp @@ -69,8 +69,7 @@ private: for (auto it = function_arguments.rbegin(); it != function_arguments.rend(); ++it) candidates.push_back({ *it, is_deterministic }); - // Using DFS we traverse function tree and try to find if it uses other keys as function arguments. - // TODO: Also process CONSTANT here. We can simplify GROUP BY x, x + 1 to GROUP BY x. + /// Using DFS we traverse function tree and try to find if it uses other keys as function arguments. while (!candidates.empty()) { auto [candidate, parents_are_only_deterministic] = candidates.back(); @@ -108,6 +107,7 @@ private: return false; } } + return true; } diff --git a/src/Analyzer/Passes/QueryAnalysisPass.cpp b/src/Analyzer/Passes/QueryAnalysisPass.cpp index f8645f2b756..f5f577a20ab 100644 --- a/src/Analyzer/Passes/QueryAnalysisPass.cpp +++ b/src/Analyzer/Passes/QueryAnalysisPass.cpp @@ -193,14 +193,9 @@ namespace ErrorCodes * lookup should not be continued, and exception must be thrown because if lookup continues identifier can be resolved from parent scope. * * TODO: Update exception messages - * TODO: JOIN TREE subquery constant columns * TODO: Table identifiers with optional UUID. * TODO: Lookup functions arrayReduce(sum, [1, 2, 3]); - * TODO: SELECT (compound_expression).*, (compound_expression).COLUMNS are not supported on parser level. - * TODO: SELECT a.b.c.*, a.b.c.COLUMNS. Qualified matcher where identifier size is greater than 2 are not supported on parser level. * TODO: Support function identifier resolve from parent query scope, if lambda in parent scope does not capture any columns. - * TODO: Support group_by_use_nulls. - * TODO: Scalar subqueries cache. */ namespace @@ -472,6 +467,12 @@ public: alias_name_to_expressions[node_alias].push_back(node); } + if (const auto * function = node->as()) + { + if (AggregateFunctionFactory::instance().isAggregateFunctionName(function->getFunctionName())) + ++aggregate_functions_counter; + } + expressions.emplace_back(node); } @@ -490,6 +491,12 @@ public: alias_name_to_expressions.erase(it); } + if (const auto * function = top_expression->as()) + { + if (AggregateFunctionFactory::instance().isAggregateFunctionName(function->getFunctionName())) + --aggregate_functions_counter; + } + expressions.pop_back(); } @@ -508,6 +515,11 @@ public: return alias_name_to_expressions.contains(alias); } + bool hasAggregateFunction() const + { + return aggregate_functions_counter > 0; + } + QueryTreeNodePtr getExpressionWithAlias(const std::string & alias) const { auto expression_it = alias_name_to_expressions.find(alias); @@ -554,6 +566,7 @@ public: private: QueryTreeNodes expressions; + size_t aggregate_functions_counter = 0; std::unordered_map alias_name_to_expressions; }; @@ -684,9 +697,15 @@ struct IdentifierResolveScope } if (auto * union_node = scope_node->as()) + { context = union_node->getContext(); + } else if (auto * query_node = scope_node->as()) + { context = query_node->getContext(); + group_by_use_nulls = context->getSettingsRef().group_by_use_nulls && + (query_node->isGroupByWithGroupingSets() || query_node->isGroupByWithRollup() || query_node->isGroupByWithCube()); + } } QueryTreeNodePtr scope_node; @@ -734,9 +753,14 @@ struct IdentifierResolveScope /// Table expression node to data std::unordered_map table_expression_node_to_data; + QueryTreeNodePtrWithHashSet nullable_group_by_keys; + /// Use identifier lookup to result cache bool use_identifier_lookup_to_result_cache = true; + /// Apply nullability to aggregation keys + bool group_by_use_nulls = false; + /// JOINs count size_t joins_count = 0; @@ -1310,6 +1334,9 @@ private: /// Global resolve expression node to projection names map std::unordered_map resolved_expressions; + /// Global resolve expression node to tree size + std::unordered_map node_to_tree_size; + /// Global scalar subquery to scalar value map std::unordered_map scalar_subquery_to_scalar_value; @@ -1838,7 +1865,10 @@ void QueryAnalyzer::evaluateScalarSubqueryIfNeeded(QueryTreeNodePtr & node, Iden Block scalar_block; - QueryTreeNodePtrWithHash node_with_hash(node); + auto node_without_alias = node->clone(); + node_without_alias->removeAlias(); + + QueryTreeNodePtrWithHash node_with_hash(node_without_alias); auto scalar_value_it = scalar_subquery_to_scalar_value.find(node_with_hash); if (scalar_value_it != scalar_subquery_to_scalar_value.end()) @@ -1928,21 +1958,7 @@ void QueryAnalyzer::evaluateScalarSubqueryIfNeeded(QueryTreeNodePtr & node, Iden * * Example: SELECT (SELECT 2 AS x, x) */ - NameSet block_column_names; - size_t unique_column_name_counter = 1; - - for (auto & column_with_type : block) - { - if (!block_column_names.contains(column_with_type.name)) - { - block_column_names.insert(column_with_type.name); - continue; - } - - column_with_type.name += '_'; - column_with_type.name += std::to_string(unique_column_name_counter); - ++unique_column_name_counter; - } + makeUniqueColumnNamesInBlock(block); scalar_block.insert({ ColumnTuple::create(block.getColumns()), @@ -2322,7 +2338,13 @@ QueryTreeNodePtr QueryAnalyzer::tryResolveTableIdentifierFromDatabaseCatalog(con storage_id = context->resolveStorageID(storage_id); bool is_temporary_table = storage_id.getDatabaseName() == DatabaseCatalog::TEMPORARY_DATABASE; - auto storage = DatabaseCatalog::instance().tryGetTable(storage_id, context); + StoragePtr storage; + + if (is_temporary_table) + storage = DatabaseCatalog::instance().getTable(storage_id, context); + else + storage = DatabaseCatalog::instance().tryGetTable(storage_id, context); + if (!storage) return {}; @@ -2888,7 +2910,10 @@ QueryTreeNodePtr QueryAnalyzer::tryResolveIdentifierFromTableExpression(const Id break; IdentifierLookup column_identifier_lookup = {qualified_identifier_with_removed_part, IdentifierLookupContext::EXPRESSION}; - if (tryBindIdentifierToAliases(column_identifier_lookup, scope) || + if (tryBindIdentifierToAliases(column_identifier_lookup, scope)) + break; + + if (table_expression_data.should_qualify_columns && tryBindIdentifierToTableExpressions(column_identifier_lookup, table_expression_node, scope)) break; @@ -2992,11 +3017,39 @@ QueryTreeNodePtr QueryAnalyzer::tryResolveIdentifierFromJoin(const IdentifierLoo resolved_identifier = std::move(result_column_node); } - else if (scope.joins_count == 1 && scope.context->getSettingsRef().single_join_prefer_left_table) + else if (left_resolved_identifier->isEqual(*right_resolved_identifier, IQueryTreeNode::CompareOptions{.compare_aliases = false})) { + const auto & identifier_path_part = identifier_lookup.identifier.front(); + auto * left_resolved_identifier_column = left_resolved_identifier->as(); + auto * right_resolved_identifier_column = right_resolved_identifier->as(); + + if (left_resolved_identifier_column && right_resolved_identifier_column) + { + const auto & left_column_source_alias = left_resolved_identifier_column->getColumnSource()->getAlias(); + const auto & right_column_source_alias = right_resolved_identifier_column->getColumnSource()->getAlias(); + + /** If column from right table was resolved using alias, we prefer column from right table. + * + * Example: SELECT dummy FROM system.one JOIN system.one AS A ON A.dummy = system.one.dummy; + * + * If alias is specified for left table, and alias is not specified for right table and identifier was resolved + * without using left table alias, we prefer column from right table. + * + * Example: SELECT dummy FROM system.one AS A JOIN system.one ON A.dummy = system.one.dummy; + * + * Otherwise we prefer column from left table. + */ + if (identifier_path_part == right_column_source_alias) + return right_resolved_identifier; + else if (!left_column_source_alias.empty() && + right_column_source_alias.empty() && + identifier_path_part != left_column_source_alias) + return right_resolved_identifier; + } + return left_resolved_identifier; } - else if (left_resolved_identifier->isEqual(*right_resolved_identifier, IQueryTreeNode::CompareOptions{.compare_aliases = false})) + else if (scope.joins_count == 1 && scope.context->getSettingsRef().single_join_prefer_left_table) { return left_resolved_identifier; } @@ -4440,6 +4493,7 @@ ProjectionNames QueryAnalyzer::resolveFunction(QueryTreeNodePtr & node, Identifi bool is_special_function_dict_get = false; bool is_special_function_join_get = false; bool is_special_function_exists = false; + bool is_special_function_if = false; if (!lambda_expression_untyped) { @@ -4447,6 +4501,7 @@ ProjectionNames QueryAnalyzer::resolveFunction(QueryTreeNodePtr & node, Identifi is_special_function_dict_get = functionIsDictGet(function_name); is_special_function_join_get = functionIsJoinGet(function_name); is_special_function_exists = function_name == "exists"; + is_special_function_if = function_name == "if"; auto function_name_lowercase = Poco::toLower(function_name); @@ -4545,6 +4600,60 @@ ProjectionNames QueryAnalyzer::resolveFunction(QueryTreeNodePtr & node, Identifi is_special_function_in = true; } + if (is_special_function_if && !function_node_ptr->getArguments().getNodes().empty()) + { + /** Handle special case with constant If function, even if some of the arguments are invalid. + * + * SELECT if(hasColumnInTable('system', 'numbers', 'not_existing_column'), not_existing_column, 5) FROM system.numbers; + */ + auto & if_function_arguments = function_node_ptr->getArguments().getNodes(); + auto if_function_condition = if_function_arguments[0]; + resolveExpressionNode(if_function_condition, scope, false /*allow_lambda_expression*/, false /*allow_table_expression*/); + + auto constant_condition = tryExtractConstantFromConditionNode(if_function_condition); + + if (constant_condition.has_value() && if_function_arguments.size() == 3) + { + QueryTreeNodePtr constant_if_result_node; + QueryTreeNodePtr possibly_invalid_argument_node; + + if (*constant_condition) + { + possibly_invalid_argument_node = if_function_arguments[2]; + constant_if_result_node = if_function_arguments[1]; + } + else + { + possibly_invalid_argument_node = if_function_arguments[1]; + constant_if_result_node = if_function_arguments[2]; + } + + bool apply_constant_if_optimization = false; + + try + { + resolveExpressionNode(possibly_invalid_argument_node, + scope, + false /*allow_lambda_expression*/, + false /*allow_table_expression*/); + } + catch (...) + { + apply_constant_if_optimization = true; + } + + if (apply_constant_if_optimization) + { + auto result_projection_names = resolveExpressionNode(constant_if_result_node, + scope, + false /*allow_lambda_expression*/, + false /*allow_table_expression*/); + node = std::move(constant_if_result_node); + return result_projection_names; + } + } + } + /// Resolve function arguments bool allow_table_expressions = is_special_function_in; @@ -5033,7 +5142,7 @@ ProjectionNames QueryAnalyzer::resolveFunction(QueryTreeNodePtr & node, Identifi /// Do not constant fold get scalar functions bool disable_constant_folding = function_name == "__getScalar" || function_name == "shardNum" || - function_name == "shardCount"; + function_name == "shardCount" || function_name == "hostName"; /** If function is suitable for constant folding try to convert it to constant. * Example: SELECT plus(1, 1); @@ -5059,7 +5168,8 @@ ProjectionNames QueryAnalyzer::resolveFunction(QueryTreeNodePtr & node, Identifi /** Do not perform constant folding if there are aggregate or arrayJoin functions inside function. * Example: SELECT toTypeName(sum(number)) FROM numbers(10); */ - if (column && isColumnConst(*column) && (!hasAggregateFunctionNodes(node) && !hasFunctionNode(node, "arrayJoin"))) + if (column && isColumnConst(*column) && !typeid_cast(column.get())->getDataColumn().isDummy() && + (!hasAggregateFunctionNodes(node) && !hasFunctionNode(node, "arrayJoin"))) { /// Replace function node with result constant node Field column_constant_value; @@ -5407,10 +5517,18 @@ ProjectionNames QueryAnalyzer::resolveExpressionNode(QueryTreeNodePtr & node, Id } } + validateTreeSize(node, scope.context->getSettingsRef().max_expanded_ast_elements, node_to_tree_size); + + if (scope.nullable_group_by_keys.contains(node) && !scope.expressions_in_resolve_process_stack.hasAggregateFunction()) + { + node = node->clone(); + node->convertToNullable(); + } + /** Update aliases after expression node was resolved. * Do not update node in alias table if we resolve it for duplicate alias. */ - if (!node_alias.empty() && use_alias_table) + if (!node_alias.empty() && use_alias_table && !scope.group_by_use_nulls) { auto it = scope.alias_name_to_expression_node.find(node_alias); if (it != scope.alias_name_to_expression_node.end()) @@ -5609,8 +5727,27 @@ void QueryAnalyzer::resolveInterpolateColumnsNodeList(QueryTreeNodePtr & interpo { auto & interpolate_node_typed = interpolate_node->as(); + auto * column_to_interpolate = interpolate_node_typed.getExpression()->as(); + if (!column_to_interpolate) + throw Exception(ErrorCodes::LOGICAL_ERROR, "INTERPOLATE can work only for indentifiers, but {} is found", + interpolate_node_typed.getExpression()->formatASTForErrorMessage()); + auto column_to_interpolate_name = column_to_interpolate->getIdentifier().getFullName(); + resolveExpressionNode(interpolate_node_typed.getExpression(), scope, false /*allow_lambda_expression*/, false /*allow_table_expression*/); - resolveExpressionNode(interpolate_node_typed.getInterpolateExpression(), scope, false /*allow_lambda_expression*/, false /*allow_table_expression*/); + + bool is_column_constant = interpolate_node_typed.getExpression()->getNodeType() == QueryTreeNodeType::CONSTANT; + + auto & interpolation_to_resolve = interpolate_node_typed.getInterpolateExpression(); + IdentifierResolveScope interpolate_scope(interpolation_to_resolve, &scope /*parent_scope*/); + + auto fake_column_node = std::make_shared(NameAndTypePair(column_to_interpolate_name, interpolate_node_typed.getExpression()->getResultType()), interpolate_node_typed.getExpression()); + if (is_column_constant) + interpolate_scope.expression_argument_name_to_node.emplace(column_to_interpolate_name, fake_column_node); + + resolveExpressionNode(interpolation_to_resolve, interpolate_scope, false /*allow_lambda_expression*/, false /*allow_table_expression*/); + + if (is_column_constant) + interpolation_to_resolve = interpolation_to_resolve->cloneAndReplace(fake_column_node, interpolate_node_typed.getExpression()); } } @@ -6418,9 +6555,6 @@ void QueryAnalyzer::resolveQuery(const QueryTreeNodePtr & query_node, Identifier auto & query_node_typed = query_node->as(); const auto & settings = scope.context->getSettingsRef(); - if (settings.group_by_use_nulls) - throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "GROUP BY use nulls is not supported"); - bool is_rollup_or_cube = query_node_typed.isGroupByWithRollup() || query_node_typed.isGroupByWithCube(); if (query_node_typed.isGroupByWithGroupingSets() && query_node_typed.isGroupByWithTotals()) @@ -6556,15 +6690,21 @@ void QueryAnalyzer::resolveQuery(const QueryTreeNodePtr & query_node, Identifier resolveQueryJoinTreeNode(query_node_typed.getJoinTree(), scope, visitor); } - scope.use_identifier_lookup_to_result_cache = true; + if (!scope.group_by_use_nulls) + scope.use_identifier_lookup_to_result_cache = true; /// Resolve query node sections. - auto projection_columns = resolveProjectionExpressionNodeList(query_node_typed.getProjectionNode(), scope); - if (query_node_typed.getProjection().getNodes().empty()) - throw Exception(ErrorCodes::EMPTY_LIST_OF_COLUMNS_QUERIED, - "Empty list of columns in projection. In scope {}", - scope.scope_node->formatASTForErrorMessage()); + NamesAndTypes projection_columns; + + if (!scope.group_by_use_nulls) + { + projection_columns = resolveProjectionExpressionNodeList(query_node_typed.getProjectionNode(), scope); + if (query_node_typed.getProjection().getNodes().empty()) + throw Exception(ErrorCodes::EMPTY_LIST_OF_COLUMNS_QUERIED, + "Empty list of columns in projection. In scope {}", + scope.scope_node->formatASTForErrorMessage()); + } if (query_node_typed.hasWith()) resolveExpressionNodeList(query_node_typed.getWithNode(), scope, true /*allow_lambda_expression*/, false /*allow_table_expression*/); @@ -6586,6 +6726,15 @@ void QueryAnalyzer::resolveQuery(const QueryTreeNodePtr & query_node, Identifier resolveExpressionNodeList(grouping_sets_keys_list_node, scope, false /*allow_lambda_expression*/, false /*allow_table_expression*/); } + + if (scope.group_by_use_nulls) + { + for (const auto & grouping_set : query_node_typed.getGroupBy().getNodes()) + { + for (const auto & group_by_elem : grouping_set->as()->getNodes()) + scope.nullable_group_by_keys.insert(group_by_elem); + } + } } else { @@ -6593,6 +6742,12 @@ void QueryAnalyzer::resolveQuery(const QueryTreeNodePtr & query_node, Identifier replaceNodesWithPositionalArguments(query_node_typed.getGroupByNode(), query_node_typed.getProjection().getNodes(), scope); resolveExpressionNodeList(query_node_typed.getGroupByNode(), scope, false /*allow_lambda_expression*/, false /*allow_table_expression*/); + + if (scope.group_by_use_nulls) + { + for (const auto & group_by_elem : query_node_typed.getGroupBy().getNodes()) + scope.nullable_group_by_keys.insert(group_by_elem); + } } } @@ -6645,6 +6800,15 @@ void QueryAnalyzer::resolveQuery(const QueryTreeNodePtr & query_node, Identifier convertLimitOffsetExpression(query_node_typed.getOffset(), "OFFSET", scope); } + if (scope.group_by_use_nulls) + { + projection_columns = resolveProjectionExpressionNodeList(query_node_typed.getProjectionNode(), scope); + if (query_node_typed.getProjection().getNodes().empty()) + throw Exception(ErrorCodes::EMPTY_LIST_OF_COLUMNS_QUERIED, + "Empty list of columns in projection. In scope {}", + scope.scope_node->formatASTForErrorMessage()); + } + /** Resolve nodes with duplicate aliases. * Table expressions cannot have duplicate aliases. * @@ -6708,7 +6872,16 @@ void QueryAnalyzer::resolveQuery(const QueryTreeNodePtr & query_node, Identifier "ARRAY JOIN", "in PREWHERE"); - validateAggregates(query_node); + validateAggregates(query_node, { .group_by_use_nulls = scope.group_by_use_nulls }); + + for (const auto & column : projection_columns) + { + if (isNotCreatable(column.type)) + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Invalid projection column with type {}. In scope {}", + column.type->getName(), + scope.scope_node->formatASTForErrorMessage()); + } /** WITH section can be safely removed, because WITH section only can provide aliases to query expressions * and CTE for other sections to use. diff --git a/src/Analyzer/Passes/ShardNumColumnToFunctionPass.h b/src/Analyzer/Passes/ShardNumColumnToFunctionPass.h index 83b974954fa..71a038bcf39 100644 --- a/src/Analyzer/Passes/ShardNumColumnToFunctionPass.h +++ b/src/Analyzer/Passes/ShardNumColumnToFunctionPass.h @@ -6,6 +6,9 @@ namespace DB { /** Rewrite _shard_num column into shardNum() function. + * + * Example: SELECT _shard_num FROM distributed_table; + * Result: SELECT shardNum() FROM distributed_table; */ class ShardNumColumnToFunctionPass final : public IQueryTreePass { diff --git a/src/Analyzer/QueryTreeBuilder.cpp b/src/Analyzer/QueryTreeBuilder.cpp index c7b9f9aae08..7dd988619ac 100644 --- a/src/Analyzer/QueryTreeBuilder.cpp +++ b/src/Analyzer/QueryTreeBuilder.cpp @@ -355,21 +355,67 @@ QueryTreeNodePtr QueryTreeBuilder::buildSelectExpression(const ASTPtr & select_q if (select_limit_by) current_query_tree->getLimitByNode() = buildExpressionList(select_limit_by, current_context); - /// Combine limit expression with limit setting + /// Combine limit expression with limit and offset settings into final limit expression + /// The sequence of application is the following - offset expression, limit expression, offset setting, limit setting. + /// Since offset setting is applied after limit expression, but we want to transfer settings into expression + /// we must decrease limit expression by offset setting and then add offset setting to offset expression. + /// select_limit - limit expression + /// limit - limit setting + /// offset - offset setting + /// + /// if select_limit + /// -- if offset >= select_limit (expr 0) + /// then (0) (0 rows) + /// -- else if limit > 0 (expr 1) + /// then min(select_limit - offset, limit) (expr 2) + /// -- else + /// then (select_limit - offset) (expr 3) + /// else if limit > 0 + /// then limit + /// + /// offset = offset + of_expr auto select_limit = select_query_typed.limitLength(); - if (select_limit && limit) + if (select_limit) { - auto function_node = std::make_shared("least"); - function_node->getArguments().getNodes().push_back(buildExpression(select_limit, current_context)); - function_node->getArguments().getNodes().push_back(std::make_shared(limit)); - current_query_tree->getLimit() = std::move(function_node); - } - else if (limit) - current_query_tree->getLimit() = std::make_shared(limit); - else if (select_limit) - current_query_tree->getLimit() = buildExpression(select_limit, current_context); + /// Shortcut + if (offset == 0 && limit == 0) + { + current_query_tree->getLimit() = buildExpression(select_limit, current_context); + } + else + { + /// expr 3 + auto expr_3 = std::make_shared("minus"); + expr_3->getArguments().getNodes().push_back(buildExpression(select_limit, current_context)); + expr_3->getArguments().getNodes().push_back(std::make_shared(offset)); - /// Combine offset expression with offset setting + /// expr 2 + auto expr_2 = std::make_shared("least"); + expr_2->getArguments().getNodes().push_back(expr_3->clone()); + expr_2->getArguments().getNodes().push_back(std::make_shared(limit)); + + /// expr 0 + auto expr_0 = std::make_shared("greaterOrEquals"); + expr_0->getArguments().getNodes().push_back(std::make_shared(offset)); + expr_0->getArguments().getNodes().push_back(buildExpression(select_limit, current_context)); + + /// expr 1 + auto expr_1 = std::make_shared(limit > 0); + + auto function_node = std::make_shared("multiIf"); + function_node->getArguments().getNodes().push_back(expr_0); + function_node->getArguments().getNodes().push_back(std::make_shared(0)); + function_node->getArguments().getNodes().push_back(expr_1); + function_node->getArguments().getNodes().push_back(expr_2); + function_node->getArguments().getNodes().push_back(expr_3); + + current_query_tree->getLimit() = std::move(function_node); + } + } + else if (limit > 0) + current_query_tree->getLimit() = std::make_shared(limit); + + /// Combine offset expression with offset setting into final offset expression auto select_offset = select_query_typed.limitOffset(); if (select_offset && offset) { diff --git a/src/Analyzer/QueryTreePassManager.cpp b/src/Analyzer/QueryTreePassManager.cpp index 2cecf4f81a2..9c0f2381c31 100644 --- a/src/Analyzer/QueryTreePassManager.cpp +++ b/src/Analyzer/QueryTreePassManager.cpp @@ -38,6 +38,7 @@ #include #include #include +#include #include #include @@ -147,7 +148,6 @@ private: /** ClickHouse query tree pass manager. * - * TODO: Support logical expressions optimizer. * TODO: Support setting convert_query_to_cnf. * TODO: Support setting optimize_using_constraints. * TODO: Support setting optimize_substitute_columns. @@ -262,6 +262,8 @@ void addQueryTreePasses(QueryTreePassManager & manager) manager.addPass(std::make_unique()); + manager.addPass(std::make_unique()); + manager.addPass(std::make_unique()); manager.addPass(std::make_unique()); manager.addPass(std::make_unique()); diff --git a/src/Analyzer/TableNode.cpp b/src/Analyzer/TableNode.cpp index a746986be04..f315d372bc9 100644 --- a/src/Analyzer/TableNode.cpp +++ b/src/Analyzer/TableNode.cpp @@ -61,12 +61,17 @@ bool TableNode::isEqualImpl(const IQueryTreeNode & rhs) const void TableNode::updateTreeHashImpl(HashState & state) const { - auto full_name = storage_id.getFullNameNotQuoted(); - state.update(full_name.size()); - state.update(full_name); - - state.update(temporary_table_name.size()); - state.update(temporary_table_name); + if (!temporary_table_name.empty()) + { + state.update(temporary_table_name.size()); + state.update(temporary_table_name); + } + else + { + auto full_name = storage_id.getFullNameNotQuoted(); + state.update(full_name.size()); + state.update(full_name); + } if (table_expression_modifiers) table_expression_modifiers->updateTreeHash(state); diff --git a/src/Analyzer/Utils.cpp b/src/Analyzer/Utils.cpp index c5a5c042cbc..eb7aceef1e8 100644 --- a/src/Analyzer/Utils.cpp +++ b/src/Analyzer/Utils.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -32,6 +33,7 @@ namespace DB namespace ErrorCodes { extern const int LOGICAL_ERROR; + extern const int BAD_ARGUMENTS; } bool isNodePartOfTree(const IQueryTreeNode * node, const IQueryTreeNode * root) @@ -79,6 +81,75 @@ bool isNameOfInFunction(const std::string & function_name) return is_special_function_in; } +bool isNameOfLocalInFunction(const std::string & function_name) +{ + bool is_special_function_in = function_name == "in" || + function_name == "notIn" || + function_name == "nullIn" || + function_name == "notNullIn" || + function_name == "inIgnoreSet" || + function_name == "notInIgnoreSet" || + function_name == "nullInIgnoreSet" || + function_name == "notNullInIgnoreSet"; + + return is_special_function_in; +} + +bool isNameOfGlobalInFunction(const std::string & function_name) +{ + bool is_special_function_in = function_name == "globalIn" || + function_name == "globalNotIn" || + function_name == "globalNullIn" || + function_name == "globalNotNullIn" || + function_name == "globalInIgnoreSet" || + function_name == "globalNotInIgnoreSet" || + function_name == "globalNullInIgnoreSet" || + function_name == "globalNotNullInIgnoreSet"; + + return is_special_function_in; +} + +std::string getGlobalInFunctionNameForLocalInFunctionName(const std::string & function_name) +{ + if (function_name == "in") + return "globalIn"; + else if (function_name == "notIn") + return "globalNotIn"; + else if (function_name == "nullIn") + return "globalNullIn"; + else if (function_name == "notNullIn") + return "globalNotNullIn"; + else if (function_name == "inIgnoreSet") + return "globalInIgnoreSet"; + else if (function_name == "notInIgnoreSet") + return "globalNotInIgnoreSet"; + else if (function_name == "nullInIgnoreSet") + return "globalNullInIgnoreSet"; + else if (function_name == "notNullInIgnoreSet") + return "globalNotNullInIgnoreSet"; + + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid local IN function name {}", function_name); +} + +void makeUniqueColumnNamesInBlock(Block & block) +{ + NameSet block_column_names; + size_t unique_column_name_counter = 1; + + for (auto & column_with_type : block) + { + if (!block_column_names.contains(column_with_type.name)) + { + block_column_names.insert(column_with_type.name); + continue; + } + + column_with_type.name += '_'; + column_with_type.name += std::to_string(unique_column_name_counter); + ++unique_column_name_counter; + } +} + QueryTreeNodePtr buildCastFunction(const QueryTreeNodePtr & expression, const DataTypePtr & type, const ContextPtr & context, @@ -102,6 +173,27 @@ QueryTreeNodePtr buildCastFunction(const QueryTreeNodePtr & expression, return cast_function_node; } +std::optional tryExtractConstantFromConditionNode(const QueryTreeNodePtr & condition_node) +{ + const auto * constant_node = condition_node->as(); + if (!constant_node) + return {}; + + const auto & value = constant_node->getValue(); + auto constant_type = constant_node->getResultType(); + constant_type = removeNullable(removeLowCardinality(constant_type)); + + auto which_constant_type = WhichDataType(constant_type); + if (!which_constant_type.isUInt8() && !which_constant_type.isNothing()) + return {}; + + if (value.isNull()) + return false; + + UInt8 predicate_value = value.safeGet(); + return predicate_value > 0; +} + static ASTPtr convertIntoTableExpressionAST(const QueryTreeNodePtr & table_expression_node) { ASTPtr table_expression_node_ast; diff --git a/src/Analyzer/Utils.h b/src/Analyzer/Utils.h index 3e2d95c6012..5802c86c462 100644 --- a/src/Analyzer/Utils.h +++ b/src/Analyzer/Utils.h @@ -13,6 +13,18 @@ bool isNodePartOfTree(const IQueryTreeNode * node, const IQueryTreeNode * root); /// Returns true if function name is name of IN function or its variations, false otherwise bool isNameOfInFunction(const std::string & function_name); +/// Returns true if function name is name of local IN function or its variations, false otherwise +bool isNameOfLocalInFunction(const std::string & function_name); + +/// Returns true if function name is name of global IN function or its variations, false otherwise +bool isNameOfGlobalInFunction(const std::string & function_name); + +/// Returns global IN function name for local IN function name +std::string getGlobalInFunctionNameForLocalInFunctionName(const std::string & function_name); + +/// Add unique suffix to names of duplicate columns in block +void makeUniqueColumnNamesInBlock(Block & block); + /** Build cast function that cast expression into type. * If resolve = true, then result cast function is resolved during build, otherwise * result cast function is not resolved during build. @@ -22,6 +34,9 @@ QueryTreeNodePtr buildCastFunction(const QueryTreeNodePtr & expression, const ContextPtr & context, bool resolve = true); +/// Try extract boolean constant from condition node +std::optional tryExtractConstantFromConditionNode(const QueryTreeNodePtr & condition_node); + /** Add table expression in tables in select query children. * If table expression node is not of identifier node, table node, query node, table function node, join node or array join node type throws logical error exception. */ diff --git a/src/Analyzer/ValidationUtils.cpp b/src/Analyzer/ValidationUtils.cpp index feb2f8a890b..58e6f26c03a 100644 --- a/src/Analyzer/ValidationUtils.cpp +++ b/src/Analyzer/ValidationUtils.cpp @@ -16,6 +16,7 @@ namespace ErrorCodes { extern const int NOT_AN_AGGREGATE; extern const int NOT_IMPLEMENTED; + extern const int BAD_ARGUMENTS; } class ValidateGroupByColumnsVisitor : public ConstInDepthQueryTreeVisitor @@ -55,7 +56,7 @@ public: } if (!found_argument_in_group_by_keys) - throw Exception(ErrorCodes::NOT_AN_AGGREGATE, + throw Exception(ErrorCodes::BAD_ARGUMENTS, "GROUPING function argument {} is not in GROUP BY keys. In query {}", grouping_function_arguments_node->formatASTForErrorMessage(), query_node->formatASTForErrorMessage()); @@ -105,7 +106,7 @@ private: const QueryTreeNodePtr & query_node; }; -void validateAggregates(const QueryTreeNodePtr & query_node) +void validateAggregates(const QueryTreeNodePtr & query_node, ValidationParams params) { const auto & query_node_typed = query_node->as(); auto join_tree_node_type = query_node_typed.getJoinTree()->getNodeType(); @@ -182,7 +183,9 @@ void validateAggregates(const QueryTreeNodePtr & query_node) if (grouping_set_key->as()) continue; - group_by_keys_nodes.push_back(grouping_set_key); + group_by_keys_nodes.push_back(grouping_set_key->clone()); + if (params.group_by_use_nulls) + group_by_keys_nodes.back()->convertToNullable(); } } else @@ -190,7 +193,9 @@ void validateAggregates(const QueryTreeNodePtr & query_node) if (node->as()) continue; - group_by_keys_nodes.push_back(node); + group_by_keys_nodes.push_back(node->clone()); + if (params.group_by_use_nulls) + group_by_keys_nodes.back()->convertToNullable(); } } @@ -279,4 +284,52 @@ void assertNoFunctionNodes(const QueryTreeNodePtr & node, visitor.visit(node); } +void validateTreeSize(const QueryTreeNodePtr & node, + size_t max_size, + std::unordered_map & node_to_tree_size) +{ + size_t tree_size = 0; + std::vector> nodes_to_process; + nodes_to_process.emplace_back(node, false); + + while (!nodes_to_process.empty()) + { + const auto [node_to_process, processed_children] = nodes_to_process.back(); + nodes_to_process.pop_back(); + + if (processed_children) + { + ++tree_size; + node_to_tree_size.emplace(node_to_process, tree_size); + continue; + } + + auto node_to_size_it = node_to_tree_size.find(node_to_process); + if (node_to_size_it != node_to_tree_size.end()) + { + tree_size += node_to_size_it->second; + continue; + } + + nodes_to_process.emplace_back(node_to_process, true); + + for (const auto & node_to_process_child : node_to_process->getChildren()) + { + if (!node_to_process_child) + continue; + + nodes_to_process.emplace_back(node_to_process_child, false); + } + + auto * constant_node = node_to_process->as(); + if (constant_node && constant_node->hasSourceExpression()) + nodes_to_process.emplace_back(constant_node->getSourceExpression(), false); + } + + if (tree_size > max_size) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Query tree is too big. Maximum: {}", + max_size); +} + } diff --git a/src/Analyzer/ValidationUtils.h b/src/Analyzer/ValidationUtils.h index b511dc9514b..c15a3531c8d 100644 --- a/src/Analyzer/ValidationUtils.h +++ b/src/Analyzer/ValidationUtils.h @@ -5,6 +5,11 @@ namespace DB { +struct ValidationParams +{ + bool group_by_use_nulls = false; +}; + /** Validate aggregates in query node. * * 1. Check that there are no aggregate functions and GROUPING function in JOIN TREE, WHERE, PREWHERE, in another aggregate functions. @@ -15,7 +20,7 @@ namespace DB * PROJECTION. * 5. Throws exception if there is GROUPING SETS or ROLLUP or CUBE or WITH TOTALS without aggregation. */ -void validateAggregates(const QueryTreeNodePtr & query_node); +void validateAggregates(const QueryTreeNodePtr & query_node, ValidationParams params); /** Assert that there are no function nodes with specified function name in node children. * Do not visit subqueries. @@ -26,4 +31,11 @@ void assertNoFunctionNodes(const QueryTreeNodePtr & node, std::string_view exception_function_name, std::string_view exception_place_message); +/** Validate tree size. If size of tree is greater than max size throws exception. + * Additionally for each node in tree, update node to tree size map. + */ +void validateTreeSize(const QueryTreeNodePtr & node, + size_t max_size, + std::unordered_map & node_to_tree_size); + } diff --git a/src/Analyzer/WindowNode.cpp b/src/Analyzer/WindowNode.cpp index 3e8537302e5..d516f7a58b8 100644 --- a/src/Analyzer/WindowNode.cpp +++ b/src/Analyzer/WindowNode.cpp @@ -113,11 +113,17 @@ ASTPtr WindowNode::toASTImpl() const window_definition->parent_window_name = parent_window_name; - window_definition->children.push_back(getPartitionByNode()->toAST()); - window_definition->partition_by = window_definition->children.back(); + if (hasPartitionBy()) + { + window_definition->children.push_back(getPartitionByNode()->toAST()); + window_definition->partition_by = window_definition->children.back(); + } - window_definition->children.push_back(getOrderByNode()->toAST()); - window_definition->order_by = window_definition->children.back(); + if (hasOrderBy()) + { + window_definition->children.push_back(getOrderByNode()->toAST()); + window_definition->order_by = window_definition->children.back(); + } window_definition->frame_is_default = window_frame.is_default; window_definition->frame_type = window_frame.type; diff --git a/src/Backups/BackupCoordinationRemote.cpp b/src/Backups/BackupCoordinationRemote.cpp index 8c696057755..8e43676f59c 100644 --- a/src/Backups/BackupCoordinationRemote.cpp +++ b/src/Backups/BackupCoordinationRemote.cpp @@ -6,7 +6,7 @@ #include #include #include -#include +#include #include @@ -166,13 +166,25 @@ namespace } BackupCoordinationRemote::BackupCoordinationRemote( - const String & root_zookeeper_path_, const String & backup_uuid_, zkutil::GetZooKeeper get_zookeeper_, bool is_internal_) - : root_zookeeper_path(root_zookeeper_path_) + const BackupKeeperSettings & keeper_settings_, + const String & root_zookeeper_path_, + const String & backup_uuid_, + zkutil::GetZooKeeper get_zookeeper_, + bool is_internal_) + : keeper_settings(keeper_settings_) + , root_zookeeper_path(root_zookeeper_path_) , zookeeper_path(root_zookeeper_path_ + "/backup-" + backup_uuid_) , backup_uuid(backup_uuid_) , get_zookeeper(get_zookeeper_) , is_internal(is_internal_) { + zookeeper_retries_info = ZooKeeperRetriesInfo( + "BackupCoordinationRemote", + &Poco::Logger::get("BackupCoordinationRemote"), + keeper_settings.keeper_max_retries, + keeper_settings.keeper_retry_initial_backoff_ms, + keeper_settings.keeper_retry_max_backoff_ms); + createRootNodes(); stage_sync.emplace( zookeeper_path + "/stage", [this] { return getZooKeeper(); }, &Poco::Logger::get("BackupCoordination")); @@ -486,19 +498,131 @@ void BackupCoordinationRemote::updateFileInfo(const FileInfo & file_info) std::vector BackupCoordinationRemote::getAllFileInfos() const { - auto zk = getZooKeeper(); - std::vector file_infos; - Strings escaped_names = zk->getChildren(zookeeper_path + "/file_names"); - for (const String & escaped_name : escaped_names) + /// There could be tons of files inside /file_names or /file_infos + /// Thus we use MultiRead requests for processing them + /// We also use [Zoo]Keeper retries and it should be safe, because + /// this function is called at the end after the actual copying is finished. + + auto split_vector = [](Strings && vec, size_t max_batch_size) -> std::vector { - String size_and_checksum = zk->get(zookeeper_path + "/file_names/" + escaped_name); - UInt64 size = deserializeSizeAndChecksum(size_and_checksum).first; - FileInfo file_info; - if (size) /// we don't keep FileInfos for empty files - file_info = deserializeFileInfo(zk->get(zookeeper_path + "/file_infos/" + size_and_checksum)); - file_info.file_name = unescapeForFileName(escaped_name); - file_infos.emplace_back(std::move(file_info)); + std::vector result; + size_t left_border = 0; + + auto move_to_result = [&](auto && begin, auto && end) + { + auto batch = Strings(); + batch.reserve(max_batch_size); + std::move(begin, end, std::back_inserter(batch)); + result.push_back(std::move(batch)); + }; + + if (max_batch_size == 0) + { + move_to_result(vec.begin(), vec.end()); + return result; + } + + for (size_t pos = 0; pos < vec.size(); ++pos) + { + if (pos >= left_border + max_batch_size) + { + move_to_result(vec.begin() + left_border, vec.begin() + pos); + left_border = pos; + } + } + + if (vec.begin() + left_border != vec.end()) + move_to_result(vec.begin() + left_border, vec.end()); + + return result; + }; + + std::vector batched_escaped_names; + { + ZooKeeperRetriesControl retries_ctl("getAllFileInfos::getChildren", zookeeper_retries_info); + retries_ctl.retryLoop([&]() + { + auto zk = getZooKeeper(); + batched_escaped_names = split_vector(zk->getChildren(zookeeper_path + "/file_names"), keeper_settings.batch_size_for_keeper_multiread); + }); } + + std::vector file_infos; + file_infos.reserve(batched_escaped_names.size()); + + for (auto & batch : batched_escaped_names) + { + zkutil::ZooKeeper::MultiGetResponse sizes_and_checksums; + { + Strings file_names_paths; + file_names_paths.reserve(batch.size()); + for (const String & escaped_name : batch) + file_names_paths.emplace_back(zookeeper_path + "/file_names/" + escaped_name); + + + ZooKeeperRetriesControl retries_ctl("getAllFileInfos::getSizesAndChecksums", zookeeper_retries_info); + retries_ctl.retryLoop([&] + { + auto zk = getZooKeeper(); + sizes_and_checksums = zk->get(file_names_paths); + }); + } + + Strings non_empty_file_names; + Strings non_empty_file_infos_paths; + std::vector non_empty_files_infos; + + /// Process all files and understand whether there are some empty files + /// Save non empty file names for further batch processing + { + std::vector empty_files_infos; + for (size_t i = 0; i < batch.size(); ++i) + { + auto file_name = batch[i]; + if (sizes_and_checksums[i].error != Coordination::Error::ZOK) + throw zkutil::KeeperException(sizes_and_checksums[i].error); + const auto & size_and_checksum = sizes_and_checksums[i].data; + auto size = deserializeSizeAndChecksum(size_and_checksum).first; + + if (size) + { + /// Save it later for batch processing + non_empty_file_names.emplace_back(file_name); + non_empty_file_infos_paths.emplace_back(zookeeper_path + "/file_infos/" + size_and_checksum); + continue; + } + + /// File is empty + FileInfo empty_file_info; + empty_file_info.file_name = unescapeForFileName(file_name); + empty_files_infos.emplace_back(std::move(empty_file_info)); + } + + std::move(empty_files_infos.begin(), empty_files_infos.end(), std::back_inserter(file_infos)); + } + + zkutil::ZooKeeper::MultiGetResponse non_empty_file_infos_serialized; + ZooKeeperRetriesControl retries_ctl("getAllFileInfos::getFileInfos", zookeeper_retries_info); + retries_ctl.retryLoop([&]() + { + auto zk = getZooKeeper(); + non_empty_file_infos_serialized = zk->get(non_empty_file_infos_paths); + }); + + /// Process non empty files + for (size_t i = 0; i < non_empty_file_names.size(); ++i) + { + FileInfo file_info; + if (non_empty_file_infos_serialized[i].error != Coordination::Error::ZOK) + throw zkutil::KeeperException(non_empty_file_infos_serialized[i].error); + file_info = deserializeFileInfo(non_empty_file_infos_serialized[i].data); + file_info.file_name = unescapeForFileName(non_empty_file_names[i]); + non_empty_files_infos.emplace_back(std::move(file_info)); + } + + std::move(non_empty_files_infos.begin(), non_empty_files_infos.end(), std::back_inserter(file_infos)); + } + return file_infos; } @@ -604,7 +728,7 @@ bool BackupCoordinationRemote::hasConcurrentBackups(const std::atomic &) return false; auto zk = getZooKeeper(); - std::string backup_stage_path = zookeeper_path +"/stage"; + std::string backup_stage_path = zookeeper_path + "/stage"; if (!zk->exists(root_zookeeper_path)) zk->createAncestors(root_zookeeper_path); diff --git a/src/Backups/BackupCoordinationRemote.h b/src/Backups/BackupCoordinationRemote.h index c7260bcd237..23c76f5be47 100644 --- a/src/Backups/BackupCoordinationRemote.h +++ b/src/Backups/BackupCoordinationRemote.h @@ -4,6 +4,7 @@ #include #include #include +#include namespace DB @@ -16,7 +17,20 @@ constexpr size_t MAX_ZOOKEEPER_ATTEMPTS = 10; class BackupCoordinationRemote : public IBackupCoordination { public: - BackupCoordinationRemote(const String & root_zookeeper_path_, const String & backup_uuid_, zkutil::GetZooKeeper get_zookeeper_, bool is_internal_); + struct BackupKeeperSettings + { + UInt64 keeper_max_retries; + UInt64 keeper_retry_initial_backoff_ms; + UInt64 keeper_retry_max_backoff_ms; + UInt64 batch_size_for_keeper_multiread; + }; + + BackupCoordinationRemote( + const BackupKeeperSettings & keeper_settings_, + const String & root_zookeeper_path_, + const String & backup_uuid_, + zkutil::GetZooKeeper get_zookeeper_, + bool is_internal_); ~BackupCoordinationRemote() override; void setStage(const String & current_host, const String & new_stage, const String & message) override; @@ -68,12 +82,14 @@ private: void prepareReplicatedTables() const; void prepareReplicatedAccess() const; + const BackupKeeperSettings keeper_settings; const String root_zookeeper_path; const String zookeeper_path; const String backup_uuid; const zkutil::GetZooKeeper get_zookeeper; const bool is_internal; + mutable ZooKeeperRetriesInfo zookeeper_retries_info; std::optional stage_sync; mutable std::mutex mutex; diff --git a/src/Backups/BackupIO.cpp b/src/Backups/BackupIO.cpp index 6ca0c8e7bee..cc252c2f1bd 100644 --- a/src/Backups/BackupIO.cpp +++ b/src/Backups/BackupIO.cpp @@ -1,9 +1,10 @@ #include #include -#include +#include #include + namespace DB { @@ -12,6 +13,15 @@ namespace ErrorCodes extern const int NOT_IMPLEMENTED; } +void IBackupReader::copyFileToDisk(const String & file_name, size_t size, DiskPtr destination_disk, const String & destination_path, + WriteMode write_mode, const WriteSettings & write_settings) +{ + auto read_buffer = readFile(file_name); + auto write_buffer = destination_disk->writeFile(destination_path, std::min(size, DBMS_DEFAULT_BUFFER_SIZE), write_mode, write_settings); + copyData(*read_buffer, *write_buffer, size); + write_buffer->finalize(); +} + void IBackupWriter::copyDataToFile(const CreateReadBufferFunction & create_read_buffer, UInt64 offset, UInt64 size, const String & dest_file_name) { auto read_buffer = create_read_buffer(); diff --git a/src/Backups/BackupIO.h b/src/Backups/BackupIO.h index fe2bed6aa1a..cf3d29ee51e 100644 --- a/src/Backups/BackupIO.h +++ b/src/Backups/BackupIO.h @@ -17,6 +17,8 @@ public: virtual bool fileExists(const String & file_name) = 0; virtual UInt64 getFileSize(const String & file_name) = 0; virtual std::unique_ptr readFile(const String & file_name) = 0; + virtual void copyFileToDisk(const String & file_name, size_t size, DiskPtr destination_disk, const String & destination_path, + WriteMode write_mode, const WriteSettings & write_settings); virtual DataSourceDescription getDataSourceDescription() const = 0; }; diff --git a/src/Backups/BackupIO_Disk.cpp b/src/Backups/BackupIO_Disk.cpp index 1b7202a5c28..cc6076541d0 100644 --- a/src/Backups/BackupIO_Disk.cpp +++ b/src/Backups/BackupIO_Disk.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -12,7 +13,8 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } -BackupReaderDisk::BackupReaderDisk(const DiskPtr & disk_, const String & path_) : disk(disk_), path(path_) +BackupReaderDisk::BackupReaderDisk(const DiskPtr & disk_, const String & path_) + : disk(disk_), path(path_), log(&Poco::Logger::get("BackupReaderDisk")) { } @@ -33,6 +35,21 @@ std::unique_ptr BackupReaderDisk::readFile(const String & fi return disk->readFile(path / file_name); } +void BackupReaderDisk::copyFileToDisk(const String & file_name, size_t size, DiskPtr destination_disk, const String & destination_path, + WriteMode write_mode, const WriteSettings & write_settings) +{ + if (write_mode == WriteMode::Rewrite) + { + LOG_TRACE(log, "Copying {}/{} from disk {} to {} by the disk", path, file_name, disk->getName(), destination_disk->getName()); + disk->copyFile(path / file_name, *destination_disk, destination_path, write_settings); + return; + } + + LOG_TRACE(log, "Copying {}/{} from disk {} to {} through buffers", path, file_name, disk->getName(), destination_disk->getName()); + IBackupReader::copyFileToDisk(file_name, size, destination_disk, destination_path, write_mode, write_settings); +} + + BackupWriterDisk::BackupWriterDisk(const DiskPtr & disk_, const String & path_) : disk(disk_), path(path_) { } diff --git a/src/Backups/BackupIO_Disk.h b/src/Backups/BackupIO_Disk.h index 5e5c431da7d..600e4f8ff39 100644 --- a/src/Backups/BackupIO_Disk.h +++ b/src/Backups/BackupIO_Disk.h @@ -17,11 +17,14 @@ public: bool fileExists(const String & file_name) override; UInt64 getFileSize(const String & file_name) override; std::unique_ptr readFile(const String & file_name) override; + void copyFileToDisk(const String & file_name, size_t size, DiskPtr destination_disk, const String & destination_path, + WriteMode write_mode, const WriteSettings & write_settings) override; DataSourceDescription getDataSourceDescription() const override; private: DiskPtr disk; std::filesystem::path path; + Poco::Logger * log; }; class BackupWriterDisk : public IBackupWriter diff --git a/src/Backups/BackupIO_File.cpp b/src/Backups/BackupIO_File.cpp index c010cae15de..5bf6d54928d 100644 --- a/src/Backups/BackupIO_File.cpp +++ b/src/Backups/BackupIO_File.cpp @@ -1,15 +1,18 @@ #include +#include #include #include #include #include +#include + namespace fs = std::filesystem; namespace DB { -BackupReaderFile::BackupReaderFile(const String & path_) : path(path_) +BackupReaderFile::BackupReaderFile(const String & path_) : path(path_), log(&Poco::Logger::get("BackupReaderFile")) { } @@ -30,6 +33,22 @@ std::unique_ptr BackupReaderFile::readFile(const String & fi return createReadBufferFromFileBase(path / file_name, {}); } +void BackupReaderFile::copyFileToDisk(const String & file_name, size_t size, DiskPtr destination_disk, const String & destination_path, + WriteMode write_mode, const WriteSettings & write_settings) +{ + if (destination_disk->getDataSourceDescription() == getDataSourceDescription()) + { + /// Use more optimal way. + LOG_TRACE(log, "Copying {}/{} to disk {} locally", path, file_name, destination_disk->getName()); + fs::copy(path / file_name, fullPath(destination_disk, destination_path), fs::copy_options::overwrite_existing); + return; + } + + LOG_TRACE(log, "Copying {}/{} to disk {} through buffers", path, file_name, destination_disk->getName()); + IBackupReader::copyFileToDisk(path / file_name, size, destination_disk, destination_path, write_mode, write_settings); +} + + BackupWriterFile::BackupWriterFile(const String & path_) : path(path_) { } diff --git a/src/Backups/BackupIO_File.h b/src/Backups/BackupIO_File.h index 1727323ba4e..e1f4324a39f 100644 --- a/src/Backups/BackupIO_File.h +++ b/src/Backups/BackupIO_File.h @@ -15,10 +15,13 @@ public: bool fileExists(const String & file_name) override; UInt64 getFileSize(const String & file_name) override; std::unique_ptr readFile(const String & file_name) override; + void copyFileToDisk(const String & file_name, size_t size, DiskPtr destination_disk, const String & destination_path, + WriteMode write_mode, const WriteSettings & write_settings) override; DataSourceDescription getDataSourceDescription() const override; private: std::filesystem::path path; + Poco::Logger * log; }; class BackupWriterFile : public IBackupWriter diff --git a/src/Backups/BackupIO_S3.cpp b/src/Backups/BackupIO_S3.cpp index a303a0bc2d5..0a757f94a49 100644 --- a/src/Backups/BackupIO_S3.cpp +++ b/src/Backups/BackupIO_S3.cpp @@ -2,9 +2,10 @@ #if USE_AWS_S3 #include +#include #include #include -#include +#include #include #include #include @@ -96,6 +97,7 @@ BackupReaderS3::BackupReaderS3( , client(makeS3Client(s3_uri_, access_key_id_, secret_access_key_, context_)) , read_settings(context_->getReadSettings()) , request_settings(context_->getStorageS3Settings().getSettings(s3_uri.uri.toString()).request_settings) + , log(&Poco::Logger::get("BackupReaderS3")) { request_settings.max_single_read_retries = context_->getSettingsRef().s3_max_single_read_retries; // FIXME: Avoid taking value for endpoint } @@ -127,6 +129,27 @@ std::unique_ptr BackupReaderS3::readFile(const String & file client, s3_uri.bucket, fs::path(s3_uri.key) / file_name, s3_uri.version_id, request_settings, read_settings); } +void BackupReaderS3::copyFileToDisk(const String & file_name, size_t size, DiskPtr destination_disk, const String & destination_path, + WriteMode write_mode, const WriteSettings & write_settings) +{ + LOG_TRACE(log, "Copying {} to disk {}", file_name, destination_disk->getName()); + + copyS3FileToDisk( + client, + s3_uri.bucket, + fs::path(s3_uri.key) / file_name, + s3_uri.version_id, + 0, + size, + destination_disk, + destination_path, + write_mode, + read_settings, + write_settings, + request_settings, + threadPoolCallbackRunner(BackupsIOThreadPool::get(), "BackupReaderS3")); +} + BackupWriterS3::BackupWriterS3( const S3::URI & s3_uri_, const String & access_key_id_, const String & secret_access_key_, const ContextPtr & context_) @@ -167,7 +190,7 @@ void BackupWriterS3::copyFileNative(DiskPtr src_disk, const String & src_file_na std::string src_bucket = object_storage->getObjectsNamespace(); auto file_path = fs::path(s3_uri.key) / dest_file_name; copyS3File(client, src_bucket, objects[0].absolute_path, src_offset, src_size, s3_uri.bucket, file_path, request_settings, {}, - threadPoolCallbackRunner(IOThreadPool::get(), "BackupWriterS3")); + threadPoolCallbackRunner(BackupsIOThreadPool::get(), "BackupWriterS3")); } } @@ -175,7 +198,7 @@ void BackupWriterS3::copyDataToFile( const CreateReadBufferFunction & create_read_buffer, UInt64 offset, UInt64 size, const String & dest_file_name) { copyDataToS3File(create_read_buffer, offset, size, client, s3_uri.bucket, fs::path(s3_uri.key) / dest_file_name, request_settings, {}, - threadPoolCallbackRunner(IOThreadPool::get(), "BackupWriterS3")); + threadPoolCallbackRunner(BackupsIOThreadPool::get(), "BackupWriterS3")); } BackupWriterS3::~BackupWriterS3() = default; @@ -222,7 +245,7 @@ std::unique_ptr BackupWriterS3::writeFile(const String & file_name) request_settings, std::nullopt, DBMS_DEFAULT_BUFFER_SIZE, - threadPoolCallbackRunner(IOThreadPool::get(), "BackupWriterS3")); + threadPoolCallbackRunner(BackupsIOThreadPool::get(), "BackupWriterS3")); } void BackupWriterS3::removeFile(const String & file_name) diff --git a/src/Backups/BackupIO_S3.h b/src/Backups/BackupIO_S3.h index 9c3132c5689..94e61248428 100644 --- a/src/Backups/BackupIO_S3.h +++ b/src/Backups/BackupIO_S3.h @@ -22,6 +22,8 @@ public: bool fileExists(const String & file_name) override; UInt64 getFileSize(const String & file_name) override; std::unique_ptr readFile(const String & file_name) override; + void copyFileToDisk(const String & file_name, size_t size, DiskPtr destination_disk, const String & destination_path, + WriteMode write_mode, const WriteSettings & write_settings) override; DataSourceDescription getDataSourceDescription() const override; private: @@ -29,6 +31,7 @@ private: std::shared_ptr client; ReadSettings read_settings; S3Settings::RequestSettings request_settings; + Poco::Logger * log; }; diff --git a/src/Backups/BackupImpl.cpp b/src/Backups/BackupImpl.cpp index e4c85bec496..a89dc2ff2b3 100644 --- a/src/Backups/BackupImpl.cpp +++ b/src/Backups/BackupImpl.cpp @@ -6,7 +6,7 @@ #include #include #include -#include +#include #include #include #include @@ -16,11 +16,11 @@ #include #include #include -#include #include #include #include #include +#include #include #include #include @@ -79,66 +79,6 @@ namespace } -class BackupImpl::BackupEntryFromBackupImpl : public IBackupEntry -{ -public: - BackupEntryFromBackupImpl( - const std::shared_ptr & backup_, - const String & archive_suffix_, - const String & data_file_name_, - UInt64 size_, - const UInt128 checksum_, - BackupEntryPtr base_backup_entry_ = {}) - : backup(backup_), archive_suffix(archive_suffix_), data_file_name(data_file_name_), size(size_), checksum(checksum_), - base_backup_entry(std::move(base_backup_entry_)) - { - } - - std::unique_ptr getReadBuffer() const override - { - std::unique_ptr read_buffer; - if (backup->use_archives) - read_buffer = backup->getArchiveReader(archive_suffix)->readFile(data_file_name); - else - read_buffer = backup->reader->readFile(data_file_name); - if (base_backup_entry) - { - size_t base_size = base_backup_entry->getSize(); - read_buffer = std::make_unique( - base_backup_entry->getReadBuffer(), base_size, std::move(read_buffer), size - base_size); - } - return read_buffer; - } - - UInt64 getSize() const override { return size; } - std::optional getChecksum() const override { return checksum; } - - String getFilePath() const override - { - return data_file_name; - } - - DiskPtr tryGetDiskIfExists() const override - { - return nullptr; - } - - DataSourceDescription getDataSourceDescription() const override - { - return backup->reader->getDataSourceDescription(); - } - - -private: - const std::shared_ptr backup; - const String archive_suffix; - const String data_file_name; - const UInt64 size; - const UInt128 checksum; - BackupEntryPtr base_backup_entry; -}; - - BackupImpl::BackupImpl( const String & backup_name_for_logging_, const ArchiveParams & archive_params_, @@ -317,11 +257,19 @@ void BackupImpl::writeBackupMetadata() { assert(!is_internal_backup); - Poco::AutoPtr config{new Poco::Util::XMLConfiguration()}; - config->setInt("version", CURRENT_BACKUP_VERSION); - config->setBool("deduplicate_files", deduplicate_files); - config->setString("timestamp", toString(LocalDateTime{timestamp})); - config->setString("uuid", toString(*uuid)); + checkLockFile(true); + + std::unique_ptr out; + if (use_archives) + out = getArchiveWriter("")->writeFile(".backup"); + else + out = writer->writeFile(".backup"); + + *out << ""; + *out << "" << CURRENT_BACKUP_VERSION << ""; + *out << "" << deduplicate_files << ""; + *out << "" << toString(LocalDateTime{timestamp}) << ""; + *out << "" << toString(*uuid) << ""; auto all_file_infos = coordination->getAllFileInfos(); @@ -336,8 +284,8 @@ void BackupImpl::writeBackupMetadata() if (base_backup_in_use) { - config->setString("base_backup", base_backup_info->toString()); - config->setString("base_backup_uuid", toString(*base_backup_uuid)); + *out << "" << xml << base_backup_info->toString() << ""; + *out << "" << toString(*base_backup_uuid) << ""; } } @@ -346,31 +294,32 @@ void BackupImpl::writeBackupMetadata() num_entries = 0; size_of_entries = 0; - for (size_t i = 0; i != all_file_infos.size(); ++i) + *out << ""; + for (const auto & info : all_file_infos) { - const auto & info = all_file_infos[i]; - String prefix = i ? "contents.file[" + std::to_string(i) + "]." : "contents.file."; - config->setString(prefix + "name", info.file_name); - config->setUInt64(prefix + "size", info.size); + *out << ""; + + *out << "" << xml << info.file_name << ""; + *out << "" << info.size << ""; if (info.size) { - config->setString(prefix + "checksum", hexChecksum(info.checksum)); + *out << "" << hexChecksum(info.checksum) << ""; if (info.base_size) { - config->setBool(prefix + "use_base", true); + *out << "true"; if (info.base_size != info.size) { - config->setUInt64(prefix + "base_size", info.base_size); - config->setString(prefix + "base_checksum", hexChecksum(info.base_checksum)); + *out << "" << info.base_size << ""; + *out << "" << hexChecksum(info.base_checksum) << ""; } } if (!info.data_file_name.empty() && (info.data_file_name != info.file_name)) - config->setString(prefix + "data_file", info.data_file_name); + *out << "" << xml << info.data_file_name << ""; if (!info.archive_suffix.empty()) - config->setString(prefix + "archive_suffix", info.archive_suffix); + *out << "" << xml << info.archive_suffix << ""; if (info.pos_in_archive != static_cast(-1)) - config->setUInt64(prefix + "pos_in_archive", info.pos_in_archive); + *out << "" << info.pos_in_archive << ""; } total_size += info.size; @@ -380,23 +329,16 @@ void BackupImpl::writeBackupMetadata() ++num_entries; size_of_entries += info.size - info.base_size; } + + *out << ""; } + *out << ""; - std::ostringstream stream; // STYLE_CHECK_ALLOW_STD_STRING_STREAM - config->save(stream); - String str = stream.str(); + *out << ""; - checkLockFile(true); - - std::unique_ptr out; - if (use_archives) - out = getArchiveWriter("")->writeFile(".backup"); - else - out = writer->writeFile(".backup"); - out->write(str.data(), str.size()); out->finalize(); - uncompressed_size = size_of_entries + str.size(); + uncompressed_size = size_of_entries + out->count(); } @@ -643,24 +585,22 @@ SizeAndChecksum BackupImpl::getFileSizeAndChecksum(const String & file_name) con return {info->size, info->checksum}; } -BackupEntryPtr BackupImpl::readFile(const String & file_name) const +std::unique_ptr BackupImpl::readFile(const String & file_name) const { return readFile(getFileSizeAndChecksum(file_name)); } -BackupEntryPtr BackupImpl::readFile(const SizeAndChecksum & size_and_checksum) const +std::unique_ptr BackupImpl::readFile(const SizeAndChecksum & size_and_checksum) const { - std::lock_guard lock{mutex}; if (open_mode != OpenMode::READ) throw Exception(ErrorCodes::LOGICAL_ERROR, "Backup is not opened for reading"); - ++num_read_files; - num_read_bytes += size_and_checksum.first; - - if (!size_and_checksum.first) + if (size_and_checksum.first == 0) { /// Entry's data is empty. - return std::make_unique(nullptr, 0, UInt128{0, 0}); + std::lock_guard lock{mutex}; + ++num_read_files; + return std::make_unique(static_cast(nullptr), 0); } auto info_opt = coordination->getFileInfo(size_and_checksum); @@ -675,45 +615,149 @@ BackupEntryPtr BackupImpl::readFile(const SizeAndChecksum & size_and_checksum) c const auto & info = *info_opt; + std::unique_ptr read_buffer; + std::unique_ptr base_read_buffer; + + if (info.size > info.base_size) + { + /// Make `read_buffer` if there is data for this backup entry in this backup. + if (use_archives) + { + std::shared_ptr archive_reader; + { + std::lock_guard lock{mutex}; + archive_reader = getArchiveReader(info.archive_suffix); + } + read_buffer = archive_reader->readFile(info.data_file_name); + } + else + { + read_buffer = reader->readFile(info.data_file_name); + } + } + + if (info.base_size) + { + /// Make `base_read_buffer` if there is data for this backup entry in the base backup. + if (!base_backup) + { + throw Exception( + ErrorCodes::NO_BASE_BACKUP, + "Backup {}: Entry {} is marked to be read from a base backup, but there is no base backup specified", + backup_name_for_logging, formatSizeAndChecksum(size_and_checksum)); + } + + if (!base_backup->fileExists(std::pair(info.base_size, info.base_checksum))) + { + throw Exception( + ErrorCodes::WRONG_BASE_BACKUP, + "Backup {}: Entry {} is marked to be read from a base backup, but doesn't exist there", + backup_name_for_logging, formatSizeAndChecksum(size_and_checksum)); + } + + base_read_buffer = base_backup->readFile(std::pair{info.base_size, info.base_checksum}); + } + + { + /// Update number of read files. + std::lock_guard lock{mutex}; + ++num_read_files; + num_read_bytes += info.size; + } + if (!info.base_size) { - /// Data goes completely from this backup, the base backup isn't used. - return std::make_unique( - std::static_pointer_cast(shared_from_this()), info.archive_suffix, info.data_file_name, info.size, info.checksum); + /// Data comes completely from this backup, the base backup isn't used. + return read_buffer; } - - if (!base_backup) + else if (info.size == info.base_size) { - throw Exception( - ErrorCodes::NO_BASE_BACKUP, - "Backup {}: Entry {} is marked to be read from a base backup, but there is no base backup specified", - backup_name_for_logging, formatSizeAndChecksum(size_and_checksum)); + /// Data comes completely from the base backup (nothing comes from this backup). + return base_read_buffer; } - - if (!base_backup->fileExists(std::pair(info.base_size, info.base_checksum))) + else { - throw Exception( - ErrorCodes::WRONG_BASE_BACKUP, - "Backup {}: Entry {} is marked to be read from a base backup, but doesn't exist there", - backup_name_for_logging, formatSizeAndChecksum(size_and_checksum)); - } - - auto base_entry = base_backup->readFile(std::pair{info.base_size, info.base_checksum}); - - if (info.size == info.base_size) - { - /// Data goes completely from the base backup (nothing goes from this backup). - return base_entry; - } - - { - /// The beginning of the data goes from the base backup, - /// and the ending goes from this backup. - return std::make_unique( - static_pointer_cast(shared_from_this()), info.archive_suffix, info.data_file_name, info.size, info.checksum, std::move(base_entry)); + /// The beginning of the data comes from the base backup, + /// and the ending comes from this backup. + return std::make_unique( + std::move(base_read_buffer), info.base_size, std::move(read_buffer), info.size - info.base_size); } } +size_t BackupImpl::copyFileToDisk(const String & file_name, DiskPtr destination_disk, const String & destination_path, + WriteMode write_mode, const WriteSettings & write_settings) const +{ + return copyFileToDisk(getFileSizeAndChecksum(file_name), destination_disk, destination_path, write_mode, write_settings); +} + +size_t BackupImpl::copyFileToDisk(const SizeAndChecksum & size_and_checksum, DiskPtr destination_disk, const String & destination_path, + WriteMode write_mode, const WriteSettings & write_settings) const +{ + if (open_mode != OpenMode::READ) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Backup is not opened for reading"); + + if (size_and_checksum.first == 0) + { + /// Entry's data is empty. + if (write_mode == WriteMode::Rewrite) + { + /// Just create an empty file. + destination_disk->createFile(destination_path); + } + std::lock_guard lock{mutex}; + ++num_read_files; + return 0; + } + + auto info_opt = coordination->getFileInfo(size_and_checksum); + if (!info_opt) + { + throw Exception( + ErrorCodes::BACKUP_ENTRY_NOT_FOUND, + "Backup {}: Entry {} not found in the backup", + backup_name_for_logging, + formatSizeAndChecksum(size_and_checksum)); + } + + const auto & info = *info_opt; + + bool file_copied = false; + + if (info.size && !info.base_size && !use_archives) + { + /// Data comes completely from this backup. + reader->copyFileToDisk(info.data_file_name, info.size, destination_disk, destination_path, write_mode, write_settings); + file_copied = true; + + } + else if (info.size && (info.size == info.base_size)) + { + /// Data comes completely from the base backup (nothing comes from this backup). + base_backup->copyFileToDisk(std::pair{info.base_size, info.base_checksum}, destination_disk, destination_path, write_mode, write_settings); + file_copied = true; + } + + if (file_copied) + { + /// The file is already copied, but `num_read_files` is not updated yet. + std::lock_guard lock{mutex}; + ++num_read_files; + num_read_bytes += info.size; + } + else + { + /// Use the generic way to copy data. `readFile()` will update `num_read_files`. + auto read_buffer = readFile(size_and_checksum); + auto write_buffer = destination_disk->writeFile(destination_path, std::min(info.size, DBMS_DEFAULT_BUFFER_SIZE), + write_mode, write_settings); + copyData(*read_buffer, *write_buffer, info.size); + write_buffer->finalize(); + } + + return info.size; +} + + namespace { diff --git a/src/Backups/BackupImpl.h b/src/Backups/BackupImpl.h index 4aa300d5021..c33ca7c94ad 100644 --- a/src/Backups/BackupImpl.h +++ b/src/Backups/BackupImpl.h @@ -73,8 +73,12 @@ public: UInt64 getFileSize(const String & file_name) const override; UInt128 getFileChecksum(const String & file_name) const override; SizeAndChecksum getFileSizeAndChecksum(const String & file_name) const override; - BackupEntryPtr readFile(const String & file_name) const override; - BackupEntryPtr readFile(const SizeAndChecksum & size_and_checksum) const override; + std::unique_ptr readFile(const String & file_name) const override; + std::unique_ptr readFile(const SizeAndChecksum & size_and_checksum) const override; + size_t copyFileToDisk(const String & file_name, DiskPtr destination_disk, const String & destination_path, + WriteMode write_mode, const WriteSettings & write_settings) const override; + size_t copyFileToDisk(const SizeAndChecksum & size_and_checksum, DiskPtr destination_disk, const String & destination_path, + WriteMode write_mode, const WriteSettings & write_settings) const override; void writeFile(const String & file_name, BackupEntryPtr entry) override; void finalizeWriting() override; bool supportsWritingInMultipleThreads() const override { return !use_archives; } diff --git a/src/Backups/BackupsWorker.cpp b/src/Backups/BackupsWorker.cpp index 865151cc9ec..bdcff249e7d 100644 --- a/src/Backups/BackupsWorker.cpp +++ b/src/Backups/BackupsWorker.cpp @@ -38,12 +38,14 @@ namespace Stage = BackupCoordinationStage; namespace { - std::shared_ptr makeBackupCoordination(const String & root_zk_path, const String & backup_uuid, const ContextPtr & context, bool is_internal_backup) + std::shared_ptr makeBackupCoordination(std::optional keeper_settings, String & root_zk_path, const String & backup_uuid, const ContextPtr & context, bool is_internal_backup) { if (!root_zk_path.empty()) { + if (!keeper_settings.has_value()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Parameter keeper_settings is empty while root_zk_path is not. This is bug"); auto get_zookeeper = [global_context = context->getGlobalContext()] { return global_context->getZooKeeper(); }; - return std::make_shared(root_zk_path, backup_uuid, get_zookeeper, is_internal_backup); + return std::make_shared(*keeper_settings, root_zk_path, backup_uuid, get_zookeeper, is_internal_backup); } else { @@ -169,7 +171,15 @@ OperationID BackupsWorker::startMakingBackup(const ASTPtr & query, const Context /// if it's not created here. However to handle errors better it's better to make a coordination here because this way /// if an exception will be thrown in startMakingBackup() other hosts will know about that. root_zk_path = context->getConfigRef().getString("backups.zookeeper_path", "/clickhouse/backups"); - backup_coordination = makeBackupCoordination(root_zk_path, toString(*backup_settings.backup_uuid), context, backup_settings.internal); + + BackupCoordinationRemote::BackupKeeperSettings keeper_settings + { + .keeper_max_retries = context->getSettingsRef().backup_keeper_max_retries, + .keeper_retry_initial_backoff_ms = context->getSettingsRef().backup_keeper_retry_initial_backoff_ms, + .keeper_retry_max_backoff_ms = context->getSettingsRef().backup_keeper_retry_max_backoff_ms, + .batch_size_for_keeper_multiread = context->getSettingsRef().backup_batch_size_for_keeper_multiread, + }; + backup_coordination = makeBackupCoordination(keeper_settings, root_zk_path, toString(*backup_settings.backup_uuid), context, backup_settings.internal); } auto backup_info = BackupInfo::fromAST(*backup_query->backup_name); @@ -265,10 +275,17 @@ void BackupsWorker::doBackup( context->checkAccess(required_access); String root_zk_path; - + std::optional keeper_settings; ClusterPtr cluster; if (on_cluster) { + keeper_settings = BackupCoordinationRemote::BackupKeeperSettings + { + .keeper_max_retries = context->getSettingsRef().backup_keeper_max_retries, + .keeper_retry_initial_backoff_ms = context->getSettingsRef().backup_keeper_retry_initial_backoff_ms, + .keeper_retry_max_backoff_ms = context->getSettingsRef().backup_keeper_retry_max_backoff_ms, + .batch_size_for_keeper_multiread = context->getSettingsRef().backup_batch_size_for_keeper_multiread, + }; root_zk_path = context->getConfigRef().getString("backups.zookeeper_path", "/clickhouse/backups"); backup_query->cluster = context->getMacros()->expand(backup_query->cluster); cluster = context->getCluster(backup_query->cluster); @@ -277,7 +294,7 @@ void BackupsWorker::doBackup( /// Make a backup coordination. if (!backup_coordination) - backup_coordination = makeBackupCoordination(root_zk_path, toString(*backup_settings.backup_uuid), context, backup_settings.internal); + backup_coordination = makeBackupCoordination(keeper_settings, root_zk_path, toString(*backup_settings.backup_uuid), context, backup_settings.internal); if (!allow_concurrent_backups && backup_coordination->hasConcurrentBackups(std::ref(num_active_backups))) throw Exception(ErrorCodes::CONCURRENT_ACCESS_NOT_SUPPORTED, "Concurrent backups not supported, turn on setting 'allow_concurrent_backups'"); diff --git a/src/Backups/IBackup.h b/src/Backups/IBackup.h index 208305e3f35..03fab6a25d6 100644 --- a/src/Backups/IBackup.h +++ b/src/Backups/IBackup.h @@ -1,6 +1,8 @@ #pragma once #include +#include +#include #include #include @@ -8,7 +10,10 @@ namespace DB { class IBackupEntry; +class IDisk; using BackupEntryPtr = std::shared_ptr; +using DiskPtr = std::shared_ptr; +class SeekableReadBuffer; /// Represents a backup, i.e. a storage of BackupEntries which can be accessed by their names. /// A backup can be either incremental or non-incremental. An incremental backup doesn't store @@ -95,8 +100,15 @@ public: virtual SizeAndChecksum getFileSizeAndChecksum(const String & file_name) const = 0; /// Reads an entry from the backup. - virtual BackupEntryPtr readFile(const String & file_name) const = 0; - virtual BackupEntryPtr readFile(const SizeAndChecksum & size_and_checksum) const = 0; + virtual std::unique_ptr readFile(const String & file_name) const = 0; + virtual std::unique_ptr readFile(const SizeAndChecksum & size_and_checksum) const = 0; + + /// Copies a file from the backup to a specified destination disk. Returns the number of bytes written. + virtual size_t copyFileToDisk(const String & file_name, DiskPtr destination_disk, const String & destination_path, + WriteMode write_mode = WriteMode::Rewrite, const WriteSettings & write_settings = {}) const = 0; + + virtual size_t copyFileToDisk(const SizeAndChecksum & size_and_checksum, DiskPtr destination_disk, const String & destination_path, + WriteMode write_mode = WriteMode::Rewrite, const WriteSettings & write_settings = {}) const = 0; /// Puts a new entry to the backup. virtual void writeFile(const String & file_name, BackupEntryPtr entry) = 0; diff --git a/src/Backups/IBackupCoordination.h b/src/Backups/IBackupCoordination.h index f5fa01a1530..588a20d9eeb 100644 --- a/src/Backups/IBackupCoordination.h +++ b/src/Backups/IBackupCoordination.h @@ -1,7 +1,9 @@ #pragma once -#include #include +#include +#include +#include namespace DB @@ -85,6 +87,22 @@ public: /// Position in the archive. UInt64 pos_in_archive = static_cast(-1); + + /// Note: this format doesn't allow to parse data back + /// It is useful only for debugging purposes + [[ maybe_unused ]] String describe() + { + String result; + result += fmt::format("file_name: {};\n", file_name); + result += fmt::format("size: {};\n", size); + result += fmt::format("checksum: {};\n", getHexUIntLowercase(checksum)); + result += fmt::format("base_size: {};\n", base_size); + result += fmt::format("base_checksum: {};\n", getHexUIntLowercase(checksum)); + result += fmt::format("data_file_name: {};\n", data_file_name); + result += fmt::format("archive_suffix: {};\n", archive_suffix); + result += fmt::format("pos_in_archive: {};\n", pos_in_archive); + return result; + } }; /// Adds file information. diff --git a/src/Backups/RestorerFromBackup.cpp b/src/Backups/RestorerFromBackup.cpp index b19cbaf975c..68a68379f79 100644 --- a/src/Backups/RestorerFromBackup.cpp +++ b/src/Backups/RestorerFromBackup.cpp @@ -316,7 +316,7 @@ void RestorerFromBackup::findTableInBackup(const QualifiedTableName & table_name = *root_path_in_use / "data" / escapeForFileName(table_name_in_backup.database) / escapeForFileName(table_name_in_backup.table); } - auto read_buffer = backup->readFile(*metadata_path)->getReadBuffer(); + auto read_buffer = backup->readFile(*metadata_path); String create_query_str; readStringUntilEOF(create_query_str, *read_buffer); read_buffer.reset(); @@ -410,7 +410,7 @@ void RestorerFromBackup::findDatabaseInBackup(const String & database_name_in_ba if (metadata_path) { - auto read_buffer = backup->readFile(*metadata_path)->getReadBuffer(); + auto read_buffer = backup->readFile(*metadata_path); String create_query_str; readStringUntilEOF(create_query_str, *read_buffer); read_buffer.reset(); diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp index 96aff9aa304..876b1d1906d 100644 --- a/src/Client/ClientBase.cpp +++ b/src/Client/ClientBase.cpp @@ -1360,7 +1360,7 @@ void ClientBase::sendData(Block & sample, const ColumnsDescription & columns_des throw; } - if (have_data_in_stdin) + if (have_data_in_stdin && !cancelled) sendDataFromStdin(sample, columns_description_for_query, parsed_query); } else if (parsed_insert_query->data) @@ -1370,7 +1370,7 @@ void ClientBase::sendData(Block & sample, const ColumnsDescription & columns_des try { sendDataFrom(data_in, sample, columns_description_for_query, parsed_query, have_data_in_stdin); - if (have_data_in_stdin) + if (have_data_in_stdin && !cancelled) sendDataFromStdin(sample, columns_description_for_query, parsed_query); } catch (Exception & e) @@ -1834,7 +1834,7 @@ bool ClientBase::executeMultiQuery(const String & all_queries_text) { /// disable logs if expects errors TestHint test_hint(all_queries_text); - if (test_hint.clientError() || test_hint.serverError()) + if (test_hint.hasClientErrors() || test_hint.hasServerErrors()) processTextAsSingleQuery("SET send_logs_level = 'fatal'"); } @@ -1876,17 +1876,17 @@ bool ClientBase::executeMultiQuery(const String & all_queries_text) // the query ends because we failed to parse it, so we consume // the entire line. TestHint hint(String(this_query_begin, this_query_end - this_query_begin)); - if (hint.serverError()) + if (hint.hasServerErrors()) { // Syntax errors are considered as client errors - current_exception->addMessage("\nExpected server error '{}'.", hint.serverError()); + current_exception->addMessage("\nExpected server error: {}.", hint.serverErrors()); current_exception->rethrow(); } - if (hint.clientError() != current_exception->code()) + if (!hint.hasExpectedClientError(current_exception->code())) { - if (hint.clientError()) - current_exception->addMessage("\nExpected client error: " + std::to_string(hint.clientError())); + if (hint.hasClientErrors()) + current_exception->addMessage("\nExpected client error: {}.", hint.clientErrors()); current_exception->rethrow(); } @@ -1935,37 +1935,37 @@ bool ClientBase::executeMultiQuery(const String & all_queries_text) bool error_matches_hint = true; if (have_error) { - if (test_hint.serverError()) + if (test_hint.hasServerErrors()) { if (!server_exception) { error_matches_hint = false; fmt::print(stderr, "Expected server error code '{}' but got no server error (query: {}).\n", - test_hint.serverError(), full_query); + test_hint.serverErrors(), full_query); } - else if (server_exception->code() != test_hint.serverError()) + else if (!test_hint.hasExpectedServerError(server_exception->code())) { error_matches_hint = false; fmt::print(stderr, "Expected server error code: {} but got: {} (query: {}).\n", - test_hint.serverError(), server_exception->code(), full_query); + test_hint.serverErrors(), server_exception->code(), full_query); } } - if (test_hint.clientError()) + if (test_hint.hasClientErrors()) { if (!client_exception) { error_matches_hint = false; fmt::print(stderr, "Expected client error code '{}' but got no client error (query: {}).\n", - test_hint.clientError(), full_query); + test_hint.clientErrors(), full_query); } - else if (client_exception->code() != test_hint.clientError()) + else if (!test_hint.hasExpectedClientError(client_exception->code())) { error_matches_hint = false; fmt::print(stderr, "Expected client error code '{}' but got '{}' (query: {}).\n", - test_hint.clientError(), client_exception->code(), full_query); + test_hint.clientErrors(), client_exception->code(), full_query); } } - if (!test_hint.clientError() && !test_hint.serverError()) + if (!test_hint.hasClientErrors() && !test_hint.hasServerErrors()) { // No error was expected but it still occurred. This is the // default case without test hint, doesn't need additional @@ -1975,19 +1975,19 @@ bool ClientBase::executeMultiQuery(const String & all_queries_text) } else { - if (test_hint.clientError()) + if (test_hint.hasClientErrors()) { error_matches_hint = false; fmt::print(stderr, "The query succeeded but the client error '{}' was expected (query: {}).\n", - test_hint.clientError(), full_query); + test_hint.clientErrors(), full_query); } - if (test_hint.serverError()) + if (test_hint.hasServerErrors()) { error_matches_hint = false; fmt::print(stderr, "The query succeeded but the server error '{}' was expected (query: {}).\n", - test_hint.serverError(), full_query); + test_hint.serverErrors(), full_query); } } diff --git a/src/Client/Connection.cpp b/src/Client/Connection.cpp index eea007a8608..6643a94c3bc 100644 --- a/src/Client/Connection.cpp +++ b/src/Client/Connection.cpp @@ -216,6 +216,7 @@ void Connection::disconnect() socket->close(); socket = nullptr; connected = false; + nonce.reset(); } @@ -324,6 +325,14 @@ void Connection::receiveHello() password_complexity_rules.push_back({std::move(original_pattern), std::move(exception_message)}); } } + if (server_revision >= DBMS_MIN_REVISION_WITH_INTERSERVER_SECRET_V2) + { + chassert(!nonce.has_value()); + + UInt64 read_nonce; + readIntBinary(read_nonce, *in); + nonce.emplace(read_nonce); + } } else if (packet_type == Protocol::Server::Exception) receiveException()->rethrow(); @@ -506,7 +515,7 @@ void Connection::sendQuery( bool with_pending_data, std::function) { - OpenTelemetry::SpanHolder span("Connection::sendQuery()"); + OpenTelemetry::SpanHolder span("Connection::sendQuery()", OpenTelemetry::CLIENT); span.addAttribute("clickhouse.query_id", query_id_); span.addAttribute("clickhouse.query", query); span.addAttribute("target", [this] () { return this->getHost() + ":" + std::to_string(this->getPort()); }); @@ -584,6 +593,9 @@ void Connection::sendQuery( { #if USE_SSL std::string data(salt); + // For backward compatibility + if (nonce.has_value()) + data += std::to_string(nonce.value()); data += cluster_secret; data += query; data += query_id; @@ -593,8 +605,8 @@ void Connection::sendQuery( std::string hash = encodeSHA256(data); writeStringBinary(hash, *out); #else - throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, - "Inter-server secret support is disabled, because ClickHouse was built without SSL library"); + throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, + "Inter-server secret support is disabled, because ClickHouse was built without SSL library"); #endif } else diff --git a/src/Client/Connection.h b/src/Client/Connection.h index d806c5e8b1f..b86567e2ed0 100644 --- a/src/Client/Connection.h +++ b/src/Client/Connection.h @@ -167,7 +167,10 @@ private: /// For inter-server authorization String cluster; String cluster_secret; + /// For DBMS_MIN_REVISION_WITH_INTERSERVER_SECRET String salt; + /// For DBMS_MIN_REVISION_WITH_INTERSERVER_SECRET_V2 + std::optional nonce; /// Address is resolved during the first connection (or the following reconnects) /// Use it only for logging purposes diff --git a/src/Client/HedgedConnections.cpp b/src/Client/HedgedConnections.cpp index b97f9454fa5..13e4fe75b3d 100644 --- a/src/Client/HedgedConnections.cpp +++ b/src/Client/HedgedConnections.cpp @@ -31,8 +31,6 @@ HedgedConnections::HedgedConnections( : hedged_connections_factory(pool_, &context_->getSettingsRef(), timeouts_, table_to_check_) , context(std::move(context_)) , settings(context->getSettingsRef()) - , drain_timeout(settings.drain_timeout) - , allow_changing_replica_until_first_data_packet(settings.allow_changing_replica_until_first_data_packet) , throttler(throttler_) { std::vector connections = hedged_connections_factory.getManyConnections(pool_mode); @@ -263,7 +261,7 @@ Packet HedgedConnections::drain() while (!epoll.empty()) { - ReplicaLocation location = getReadyReplicaLocation(DrainCallback{drain_timeout}); + ReplicaLocation location = getReadyReplicaLocation(); Packet packet = receivePacketFromReplica(location); switch (packet.type) { @@ -290,10 +288,10 @@ Packet HedgedConnections::drain() Packet HedgedConnections::receivePacket() { std::lock_guard lock(cancel_mutex); - return receivePacketUnlocked({}, false /* is_draining */); + return receivePacketUnlocked({}); } -Packet HedgedConnections::receivePacketUnlocked(AsyncCallback async_callback, bool /* is_draining */) +Packet HedgedConnections::receivePacketUnlocked(AsyncCallback async_callback) { if (!sent_query) throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot receive packets: no query sent."); @@ -413,7 +411,7 @@ Packet HedgedConnections::receivePacketFromReplica(const ReplicaLocation & repli { /// If we are allowed to change replica until the first data packet, /// just restart timeout (if it hasn't expired yet). Otherwise disable changing replica with this offset. - if (allow_changing_replica_until_first_data_packet && !replica.is_change_replica_timeout_expired) + if (settings.allow_changing_replica_until_first_data_packet && !replica.is_change_replica_timeout_expired) replica.change_replica_timeout.setRelative(hedged_connections_factory.getConnectionTimeouts().receive_data_timeout); else disableChangingReplica(replica_location); diff --git a/src/Client/HedgedConnections.h b/src/Client/HedgedConnections.h index 40f031a16a6..45a98a14bf0 100644 --- a/src/Client/HedgedConnections.h +++ b/src/Client/HedgedConnections.h @@ -101,7 +101,7 @@ public: Packet receivePacket() override; - Packet receivePacketUnlocked(AsyncCallback async_callback, bool is_draining) override; + Packet receivePacketUnlocked(AsyncCallback async_callback) override; void disconnect() override; @@ -196,12 +196,6 @@ private: Epoll epoll; ContextPtr context; const Settings & settings; - - /// The following two fields are from settings but can be referenced outside the lifetime of - /// settings when connection is drained asynchronously. - Poco::Timespan drain_timeout; - bool allow_changing_replica_until_first_data_packet; - ThrottlerPtr throttler; bool sent_query = false; bool cancelled = false; diff --git a/src/Client/IConnections.cpp b/src/Client/IConnections.cpp deleted file mode 100644 index 9cc5a62ce12..00000000000 --- a/src/Client/IConnections.cpp +++ /dev/null @@ -1,36 +0,0 @@ -#include -#include - -namespace DB -{ - -namespace ErrorCodes -{ - extern const int SOCKET_TIMEOUT; -} - -/// This wrapper struct allows us to use Poco's socket polling code with a raw fd. -/// The only difference from Poco::Net::SocketImpl is that we don't close the fd in the destructor. -struct PocoSocketWrapper : public Poco::Net::SocketImpl -{ - explicit PocoSocketWrapper(int fd) - { - reset(fd); - } - - // Do not close fd. - ~PocoSocketWrapper() override { reset(-1); } -}; - -void IConnections::DrainCallback::operator()(int fd, Poco::Timespan, const std::string & fd_description) const -{ - if (!PocoSocketWrapper(fd).poll(drain_timeout, Poco::Net::Socket::SELECT_READ)) - { - throw Exception(ErrorCodes::SOCKET_TIMEOUT, - "Read timeout ({} ms) while draining from {}", - drain_timeout.totalMilliseconds(), - fd_description); - } -} - -} diff --git a/src/Client/IConnections.h b/src/Client/IConnections.h index 0040eeb31ed..4c81617f8d6 100644 --- a/src/Client/IConnections.h +++ b/src/Client/IConnections.h @@ -13,12 +13,6 @@ namespace DB class IConnections : boost::noncopyable { public: - struct DrainCallback - { - Poco::Timespan drain_timeout; - void operator()(int fd, Poco::Timespan, const std::string & fd_description = "") const; - }; - /// Send all scalars to replicas. virtual void sendScalarsData(Scalars & data) = 0; /// Send all content of external tables to replicas. @@ -40,7 +34,7 @@ public: virtual Packet receivePacket() = 0; /// Version of `receivePacket` function without locking. - virtual Packet receivePacketUnlocked(AsyncCallback async_callback, bool is_draining) = 0; + virtual Packet receivePacketUnlocked(AsyncCallback async_callback) = 0; /// Break all active connections. virtual void disconnect() = 0; diff --git a/src/Client/MultiplexedConnections.cpp b/src/Client/MultiplexedConnections.cpp index cc260353339..58568462ecc 100644 --- a/src/Client/MultiplexedConnections.cpp +++ b/src/Client/MultiplexedConnections.cpp @@ -20,7 +20,7 @@ namespace ErrorCodes MultiplexedConnections::MultiplexedConnections(Connection & connection, const Settings & settings_, const ThrottlerPtr & throttler) - : settings(settings_), drain_timeout(settings.drain_timeout), receive_timeout(settings.receive_timeout) + : settings(settings_) { connection.setThrottler(throttler); @@ -33,7 +33,7 @@ MultiplexedConnections::MultiplexedConnections(Connection & connection, const Se MultiplexedConnections::MultiplexedConnections(std::shared_ptr connection_ptr_, const Settings & settings_, const ThrottlerPtr & throttler) - : settings(settings_), drain_timeout(settings.drain_timeout), receive_timeout(settings.receive_timeout) + : settings(settings_) , connection_ptr(connection_ptr_) { connection_ptr->setThrottler(throttler); @@ -46,8 +46,9 @@ MultiplexedConnections::MultiplexedConnections(std::shared_ptr conne } MultiplexedConnections::MultiplexedConnections( - std::vector && connections, const Settings & settings_, const ThrottlerPtr & throttler) - : settings(settings_), drain_timeout(settings.drain_timeout), receive_timeout(settings.receive_timeout) + std::vector && connections, + const Settings & settings_, const ThrottlerPtr & throttler) + : settings(settings_) { /// If we didn't get any connections from pool and getMany() did not throw exceptions, this means that /// `skip_unavailable_shards` was set. Then just return. @@ -206,7 +207,7 @@ void MultiplexedConnections::sendMergeTreeReadTaskResponse(const ParallelReadRes Packet MultiplexedConnections::receivePacket() { std::lock_guard lock(cancel_mutex); - Packet packet = receivePacketUnlocked({}, false /* is_draining */); + Packet packet = receivePacketUnlocked({}); return packet; } @@ -254,7 +255,7 @@ Packet MultiplexedConnections::drain() while (hasActiveConnections()) { - Packet packet = receivePacketUnlocked(DrainCallback{drain_timeout}, true /* is_draining */); + Packet packet = receivePacketUnlocked({}); switch (packet.type) { @@ -304,14 +305,14 @@ std::string MultiplexedConnections::dumpAddressesUnlocked() const return buf.str(); } -Packet MultiplexedConnections::receivePacketUnlocked(AsyncCallback async_callback, bool is_draining) +Packet MultiplexedConnections::receivePacketUnlocked(AsyncCallback async_callback) { if (!sent_query) throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot receive packets: no query sent."); if (!hasActiveConnections()) throw Exception(ErrorCodes::LOGICAL_ERROR, "No more packets are available."); - ReplicaState & state = getReplicaForReading(is_draining); + ReplicaState & state = getReplicaForReading(); current_connection = state.connection; if (current_connection == nullptr) throw Exception(ErrorCodes::NO_AVAILABLE_REPLICA, "Logical error: no available replica"); @@ -366,10 +367,9 @@ Packet MultiplexedConnections::receivePacketUnlocked(AsyncCallback async_callbac return packet; } -MultiplexedConnections::ReplicaState & MultiplexedConnections::getReplicaForReading(bool is_draining) +MultiplexedConnections::ReplicaState & MultiplexedConnections::getReplicaForReading() { - /// Fast path when we only focus on one replica and are not draining the connection. - if (replica_states.size() == 1 && !is_draining) + if (replica_states.size() == 1) return replica_states[0]; Poco::Net::Socket::SocketList read_list; @@ -390,7 +390,7 @@ MultiplexedConnections::ReplicaState & MultiplexedConnections::getReplicaForRead Poco::Net::Socket::SocketList write_list; Poco::Net::Socket::SocketList except_list; - auto timeout = is_draining ? drain_timeout : receive_timeout; + auto timeout = settings.receive_timeout; int n = 0; /// EINTR loop @@ -417,9 +417,7 @@ MultiplexedConnections::ReplicaState & MultiplexedConnections::getReplicaForRead break; } - /// We treat any error as timeout for simplicity. - /// And we also check if read_list is still empty just in case. - if (n <= 0 || read_list.empty()) + if (n == 0) { const auto & addresses = dumpAddressesUnlocked(); for (ReplicaState & state : replica_states) @@ -438,7 +436,9 @@ MultiplexedConnections::ReplicaState & MultiplexedConnections::getReplicaForRead } } - /// TODO Motivation of rand is unclear. + /// TODO Absolutely wrong code: read_list could be empty; motivation of rand is unclear. + /// This code path is disabled by default. + auto & socket = read_list[thread_local_rng() % read_list.size()]; if (fd_to_replica_state_idx.empty()) { diff --git a/src/Client/MultiplexedConnections.h b/src/Client/MultiplexedConnections.h index dd228067ed2..a26e6dafbe5 100644 --- a/src/Client/MultiplexedConnections.h +++ b/src/Client/MultiplexedConnections.h @@ -65,7 +65,7 @@ public: void setReplicaInfo(ReplicaInfo value) override { replica_info = value; } private: - Packet receivePacketUnlocked(AsyncCallback async_callback, bool is_draining) override; + Packet receivePacketUnlocked(AsyncCallback async_callback) override; /// Internal version of `dumpAddresses` function without locking. std::string dumpAddressesUnlocked() const; @@ -78,18 +78,13 @@ private: }; /// Get a replica where you can read the data. - ReplicaState & getReplicaForReading(bool is_draining); + ReplicaState & getReplicaForReading(); /// Mark the replica as invalid. void invalidateReplica(ReplicaState & replica_state); const Settings & settings; - /// The following two fields are from settings but can be referenced outside the lifetime of - /// settings when connection is drained asynchronously. - Poco::Timespan drain_timeout; - Poco::Timespan receive_timeout; - /// The current number of valid connections to the replicas of this shard. size_t active_connection_count = 0; diff --git a/src/Client/TestHint.cpp b/src/Client/TestHint.cpp index f6d1e5d73c3..b64882577ee 100644 --- a/src/Client/TestHint.cpp +++ b/src/Client/TestHint.cpp @@ -1,32 +1,15 @@ -#include "TestHint.h" +#include +#include + +#include -#include -#include -#include -#include #include +#include +#include -namespace +namespace DB::ErrorCodes { - -/// Parse error as number or as a string (name of the error code const) -int parseErrorCode(DB::ReadBufferFromString & in) -{ - int code = -1; - String code_name; - - auto * pos = in.position(); - tryReadText(code, in); - if (pos != in.position()) - { - return code; - } - - /// Try parse as string - readStringUntilWhitespace(code_name, in); - return DB::ErrorCodes::getErrorCodeByName(code_name); -} - + extern const int CANNOT_PARSE_TEXT; } namespace DB @@ -60,8 +43,8 @@ TestHint::TestHint(const String & query_) size_t pos_end = comment.find('}', pos_start); if (pos_end != String::npos) { - String hint(comment.begin() + pos_start + 1, comment.begin() + pos_end); - parse(hint, is_leading_hint); + Lexer comment_lexer(comment.c_str() + pos_start + 1, comment.c_str() + pos_end, 0); + parse(comment_lexer, is_leading_hint); } } } @@ -69,33 +52,86 @@ TestHint::TestHint(const String & query_) } } -void TestHint::parse(const String & hint, bool is_leading_hint) +bool TestHint::hasExpectedClientError(int error) { - ReadBufferFromString in(hint); - String item; + return std::find(client_errors.begin(), client_errors.end(), error) != client_errors.end(); +} - while (!in.eof()) +bool TestHint::hasExpectedServerError(int error) +{ + return std::find(server_errors.begin(), server_errors.end(), error) != server_errors.end(); +} + +void TestHint::parse(Lexer & comment_lexer, bool is_leading_hint) +{ + std::unordered_set commands{"echo", "echoOn", "echoOff"}; + + std::unordered_set command_errors{ + "serverError", + "clientError", + }; + + for (Token token = comment_lexer.nextToken(); !token.isEnd(); token = comment_lexer.nextToken()) { - readStringUntilWhitespace(item, in); - if (in.eof()) - break; - - skipWhitespaceIfAny(in); - - if (!is_leading_hint) + String item = String(token.begin, token.end); + if (token.type == TokenType::BareWord && commands.contains(item)) { - if (item == "serverError") - server_error = parseErrorCode(in); - else if (item == "clientError") - client_error = parseErrorCode(in); + if (item == "echo") + echo.emplace(true); + if (item == "echoOn") + echo.emplace(true); + if (item == "echoOff") + echo.emplace(false); } + else if (!is_leading_hint && token.type == TokenType::BareWord && command_errors.contains(item)) + { + /// Everything after this must be a list of errors separated by comma + ErrorVector error_codes; + while (!token.isEnd()) + { + token = comment_lexer.nextToken(); + if (token.type == TokenType::Whitespace) + continue; + if (token.type == TokenType::Number) + { + int code; + auto [p, ec] = std::from_chars(token.begin, token.end, code); + if (p == token.begin) + throw DB::Exception( + DB::ErrorCodes::CANNOT_PARSE_TEXT, + "Could not parse integer number for errorcode: {}", + std::string_view(token.begin, token.end)); + error_codes.push_back(code); + } + else if (token.type == TokenType::BareWord) + { + int code = DB::ErrorCodes::getErrorCodeByName(std::string_view(token.begin, token.end)); + error_codes.push_back(code); + } + else + throw DB::Exception( + DB::ErrorCodes::CANNOT_PARSE_TEXT, + "Could not parse error code in {}: {}", + getTokenName(token.type), + std::string_view(token.begin, token.end)); + do + { + token = comment_lexer.nextToken(); + } while (!token.isEnd() && token.type == TokenType::Whitespace); - if (item == "echo") - echo.emplace(true); - if (item == "echoOn") - echo.emplace(true); - if (item == "echoOff") - echo.emplace(false); + if (!token.isEnd() && token.type != TokenType::Comma) + throw DB::Exception( + DB::ErrorCodes::CANNOT_PARSE_TEXT, + "Could not parse error code. Expected ','. Got '{}'", + std::string_view(token.begin, token.end)); + } + + if (item == "serverError") + server_errors = error_codes; + else + client_errors = error_codes; + break; + } } } diff --git a/src/Client/TestHint.h b/src/Client/TestHint.h index 7fa4e86c025..63f16b1dd97 100644 --- a/src/Client/TestHint.h +++ b/src/Client/TestHint.h @@ -1,21 +1,30 @@ #pragma once #include +#include + +#include + #include namespace DB { +class Lexer; + /// Checks expected server and client error codes. /// /// The following comment hints are supported: /// /// - "-- { serverError 60 }" -- in case of you are expecting server error. +/// - "-- { serverError 16, 36 }" -- in case of you are expecting one of the 2 errors. /// /// - "-- { clientError 20 }" -- in case of you are expecting client error. +/// - "-- { clientError 20, 60, 92 }" -- It's expected that the client will return one of the 3 errors. /// /// - "-- { serverError FUNCTION_THROW_IF_VALUE_IS_NON_ZERO }" -- by error name. +/// - "-- { serverError NO_SUCH_COLUMN_IN_TABLE, BAD_ARGUMENTS }" -- by error name. /// /// - "-- { clientError FUNCTION_THROW_IF_VALUE_IS_NON_ZERO }" -- by error name. /// @@ -43,29 +52,73 @@ namespace DB class TestHint { public: + using ErrorVector = std::vector; TestHint(const String & query_); - int serverError() const { return server_error; } - int clientError() const { return client_error; } + const auto & serverErrors() const { return server_errors; } + const auto & clientErrors() const { return client_errors; } std::optional echoQueries() const { return echo; } + bool hasClientErrors() { return !client_errors.empty(); } + bool hasServerErrors() { return !server_errors.empty(); } + + bool hasExpectedClientError(int error); + bool hasExpectedServerError(int error); + private: const String & query; - int server_error = 0; - int client_error = 0; + ErrorVector server_errors{}; + ErrorVector client_errors{}; std::optional echo; - void parse(const String & hint, bool is_leading_hint); + void parse(Lexer & comment_lexer, bool is_leading_hint); bool allErrorsExpected(int actual_server_error, int actual_client_error) const { - return (server_error || client_error) && (server_error == actual_server_error) && (client_error == actual_client_error); + if (actual_server_error && std::find(server_errors.begin(), server_errors.end(), actual_server_error) == server_errors.end()) + return false; + if (!actual_server_error && server_errors.size()) + return false; + + if (actual_client_error && std::find(client_errors.begin(), client_errors.end(), actual_client_error) == client_errors.end()) + return false; + if (!actual_client_error && client_errors.size()) + return false; + + return true; } bool lostExpectedError(int actual_server_error, int actual_client_error) const { - return (server_error && !actual_server_error) || (client_error && !actual_client_error); + return (server_errors.size() && !actual_server_error) || (client_errors.size() && !actual_client_error); } }; } + +template <> +struct fmt::formatter +{ + static constexpr auto parse(format_parse_context & ctx) + { + const auto * it = ctx.begin(); + const auto * end = ctx.end(); + + /// Only support {}. + if (it != end && *it != '}') + throw format_error("Invalid format"); + + return it; + } + + template + auto format(const DB::TestHint::ErrorVector & ErrorVector, FormatContext & ctx) + { + if (ErrorVector.empty()) + return format_to(ctx.out(), "{}", 0); + else if (ErrorVector.size() == 1) + return format_to(ctx.out(), "{}", ErrorVector[0]); + else + return format_to(ctx.out(), "[{}]", fmt::join(ErrorVector, ", ")); + } +}; diff --git a/src/Columns/ColumnAggregateFunction.h b/src/Columns/ColumnAggregateFunction.h index 38040d65d3b..f9ce45708c9 100644 --- a/src/Columns/ColumnAggregateFunction.h +++ b/src/Columns/ColumnAggregateFunction.h @@ -220,7 +220,12 @@ public: double getRatioOfDefaultRows(double) const override { - throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method getRatioOfDefaultRows is not supported for ColumnAggregateFunction"); + return 0.0; + } + + UInt64 getNumberOfDefaultRows() const override + { + return 0; } void getIndicesOfNonDefaultRows(Offsets &, size_t, size_t) const override diff --git a/src/Columns/ColumnArray.cpp b/src/Columns/ColumnArray.cpp index 0d085a3275a..cbeebe52141 100644 --- a/src/Columns/ColumnArray.cpp +++ b/src/Columns/ColumnArray.cpp @@ -953,6 +953,11 @@ double ColumnArray::getRatioOfDefaultRows(double sample_ratio) const return getRatioOfDefaultRowsImpl(sample_ratio); } +UInt64 ColumnArray::getNumberOfDefaultRows() const +{ + return getNumberOfDefaultRowsImpl(); +} + void ColumnArray::getIndicesOfNonDefaultRows(Offsets & indices, size_t from, size_t limit) const { return getIndicesOfNonDefaultRowsImpl(indices, from, limit); diff --git a/src/Columns/ColumnArray.h b/src/Columns/ColumnArray.h index 44652fd0c4b..e60c388831d 100644 --- a/src/Columns/ColumnArray.h +++ b/src/Columns/ColumnArray.h @@ -173,6 +173,7 @@ public: } double getRatioOfDefaultRows(double sample_ratio) const override; + UInt64 getNumberOfDefaultRows() const override; void getIndicesOfNonDefaultRows(Offsets & indices, size_t from, size_t limit) const override; diff --git a/src/Columns/ColumnCompressed.h b/src/Columns/ColumnCompressed.h index b258dbac878..d747f8ef5f4 100644 --- a/src/Columns/ColumnCompressed.h +++ b/src/Columns/ColumnCompressed.h @@ -117,6 +117,7 @@ public: void getExtremes(Field &, Field &) const override { throwMustBeDecompressed(); } size_t byteSizeAt(size_t) const override { throwMustBeDecompressed(); } double getRatioOfDefaultRows(double) const override { throwMustBeDecompressed(); } + UInt64 getNumberOfDefaultRows() const override { throwMustBeDecompressed(); } void getIndicesOfNonDefaultRows(Offsets &, size_t, size_t) const override { throwMustBeDecompressed(); } protected: diff --git a/src/Columns/ColumnConst.h b/src/Columns/ColumnConst.h index b86ed393e44..fcdcd2ce224 100644 --- a/src/Columns/ColumnConst.h +++ b/src/Columns/ColumnConst.h @@ -253,6 +253,11 @@ public: return data->isDefaultAt(0) ? 1.0 : 0.0; } + UInt64 getNumberOfDefaultRows() const override + { + return data->isDefaultAt(0) ? s : 0; + } + void getIndicesOfNonDefaultRows(Offsets & indices, size_t from, size_t limit) const override { if (!data->isDefaultAt(0)) diff --git a/src/Columns/ColumnDecimal.h b/src/Columns/ColumnDecimal.h index d47164a178d..03e0b9be558 100644 --- a/src/Columns/ColumnDecimal.h +++ b/src/Columns/ColumnDecimal.h @@ -136,6 +136,11 @@ public: return this->template getRatioOfDefaultRowsImpl(sample_ratio); } + UInt64 getNumberOfDefaultRows() const override + { + return this->template getNumberOfDefaultRowsImpl(); + } + void getIndicesOfNonDefaultRows(IColumn::Offsets & indices, size_t from, size_t limit) const override { return this->template getIndicesOfNonDefaultRowsImpl(indices, from, limit); diff --git a/src/Columns/ColumnFixedString.h b/src/Columns/ColumnFixedString.h index 7c2d9b1a155..39497e3403e 100644 --- a/src/Columns/ColumnFixedString.h +++ b/src/Columns/ColumnFixedString.h @@ -200,6 +200,11 @@ public: return getRatioOfDefaultRowsImpl(sample_ratio); } + UInt64 getNumberOfDefaultRows() const override + { + return getNumberOfDefaultRowsImpl(); + } + void getIndicesOfNonDefaultRows(Offsets & indices, size_t from, size_t limit) const override { return getIndicesOfNonDefaultRowsImpl(indices, from, limit); diff --git a/src/Columns/ColumnFunction.h b/src/Columns/ColumnFunction.h index 7b7ceb29a10..a1f6245c2bd 100644 --- a/src/Columns/ColumnFunction.h +++ b/src/Columns/ColumnFunction.h @@ -168,6 +168,11 @@ public: throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method getRatioOfDefaultRows is not supported for {}", getName()); } + UInt64 getNumberOfDefaultRows() const override + { + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method getNumberOfDefaultRows is not supported for {}", getName()); + } + void getIndicesOfNonDefaultRows(Offsets &, size_t, size_t) const override { throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method getIndicesOfNonDefaultRows is not supported for {}", getName()); diff --git a/src/Columns/ColumnLowCardinality.h b/src/Columns/ColumnLowCardinality.h index eb7fd958a69..e7f4b92d733 100644 --- a/src/Columns/ColumnLowCardinality.h +++ b/src/Columns/ColumnLowCardinality.h @@ -199,6 +199,11 @@ public: return getIndexes().getRatioOfDefaultRows(sample_ratio); } + UInt64 getNumberOfDefaultRows() const override + { + return getIndexes().getNumberOfDefaultRows(); + } + void getIndicesOfNonDefaultRows(Offsets & indices, size_t from, size_t limit) const override { return getIndexes().getIndicesOfNonDefaultRows(indices, from, limit); diff --git a/src/Columns/ColumnMap.cpp b/src/Columns/ColumnMap.cpp index 2b5f5f94ee8..b533b68b98d 100644 --- a/src/Columns/ColumnMap.cpp +++ b/src/Columns/ColumnMap.cpp @@ -296,6 +296,11 @@ double ColumnMap::getRatioOfDefaultRows(double sample_ratio) const return getRatioOfDefaultRowsImpl(sample_ratio); } +UInt64 ColumnMap::getNumberOfDefaultRows() const +{ + return getNumberOfDefaultRowsImpl(); +} + void ColumnMap::getIndicesOfNonDefaultRows(Offsets & indices, size_t from, size_t limit) const { return getIndicesOfNonDefaultRowsImpl(indices, from, limit); diff --git a/src/Columns/ColumnMap.h b/src/Columns/ColumnMap.h index db918c3db50..a11905fcaa0 100644 --- a/src/Columns/ColumnMap.h +++ b/src/Columns/ColumnMap.h @@ -92,6 +92,7 @@ public: void forEachSubcolumnRecursively(RecursiveColumnCallback callback) const override; bool structureEquals(const IColumn & rhs) const override; double getRatioOfDefaultRows(double sample_ratio) const override; + UInt64 getNumberOfDefaultRows() const override; void getIndicesOfNonDefaultRows(Offsets & indices, size_t from, size_t limit) const override; void finalize() override { nested->finalize(); } bool isFinalized() const override { return nested->isFinalized(); } diff --git a/src/Columns/ColumnNullable.h b/src/Columns/ColumnNullable.h index 9ea0ceb1c5a..47afd982b1e 100644 --- a/src/Columns/ColumnNullable.h +++ b/src/Columns/ColumnNullable.h @@ -156,6 +156,11 @@ public: return getRatioOfDefaultRowsImpl(sample_ratio); } + UInt64 getNumberOfDefaultRows() const override + { + return getNumberOfDefaultRowsImpl(); + } + void getIndicesOfNonDefaultRows(Offsets & indices, size_t from, size_t limit) const override { getIndicesOfNonDefaultRowsImpl(indices, from, limit); diff --git a/src/Columns/ColumnObject.h b/src/Columns/ColumnObject.h index d09974fcc46..91920908542 100644 --- a/src/Columns/ColumnObject.h +++ b/src/Columns/ColumnObject.h @@ -254,6 +254,7 @@ public: bool hasEqualValues() const override { throwMustBeConcrete(); } size_t byteSizeAt(size_t) const override { throwMustBeConcrete(); } double getRatioOfDefaultRows(double) const override { throwMustBeConcrete(); } + UInt64 getNumberOfDefaultRows() const override { throwMustBeConcrete(); } void getIndicesOfNonDefaultRows(Offsets &, size_t, size_t) const override { throwMustBeConcrete(); } private: diff --git a/src/Columns/ColumnSparse.cpp b/src/Columns/ColumnSparse.cpp index 43802b6bbc8..fbab61c879d 100644 --- a/src/Columns/ColumnSparse.cpp +++ b/src/Columns/ColumnSparse.cpp @@ -450,11 +450,14 @@ void ColumnSparse::compareColumn(const IColumn & rhs, size_t rhs_row_num, { const auto & rhs_sparse = assert_cast(rhs); PaddedPODArray nested_result; - values->compareColumn(rhs_sparse.getValuesColumn(), rhs_sparse.getValueIndex(rhs_row_num), + values->compareColumn( + rhs_sparse.getValuesColumn(), + rhs_sparse.getValueIndex(rhs_row_num), nullptr, nested_result, direction, nan_direction_hint); const auto & offsets_data = getOffsetsData(); - compare_results.resize_fill(_size, nested_result[0]); + compare_results.resize(size()); + std::fill(compare_results.begin(), compare_results.end(), nested_result[0]); for (size_t i = 0; i < offsets_data.size(); ++i) compare_results[offsets_data[i]] = nested_result[i + 1]; } @@ -470,7 +473,7 @@ int ColumnSparse::compareAtWithCollation(size_t n, size_t m, const IColumn & rhs bool ColumnSparse::hasEqualValues() const { - size_t num_defaults = getNumberOfDefaults(); + size_t num_defaults = getNumberOfDefaultRows(); if (num_defaults == _size) return true; @@ -512,7 +515,7 @@ void ColumnSparse::getPermutationImpl(IColumn::PermutationSortDirection directio else values->getPermutation(direction, stability, limit + 1, null_direction_hint, perm); - size_t num_of_defaults = getNumberOfDefaults(); + size_t num_of_defaults = getNumberOfDefaultRows(); size_t row = 0; const auto & offsets_data = getOffsetsData(); @@ -677,7 +680,7 @@ void ColumnSparse::getExtremes(Field & min, Field & max) const return; } - if (getNumberOfDefaults() == 0) + if (getNumberOfDefaultRows() == 0) { size_t min_idx = 1; size_t max_idx = 1; @@ -709,7 +712,12 @@ void ColumnSparse::getIndicesOfNonDefaultRows(IColumn::Offsets & indices, size_t double ColumnSparse::getRatioOfDefaultRows(double) const { - return static_cast(getNumberOfDefaults()) / _size; + return static_cast(getNumberOfDefaultRows()) / _size; +} + +UInt64 ColumnSparse::getNumberOfDefaultRows() const +{ + return _size - offsets->size(); } MutableColumns ColumnSparse::scatter(ColumnIndex num_columns, const Selector & selector) const diff --git a/src/Columns/ColumnSparse.h b/src/Columns/ColumnSparse.h index ffe392a83c1..d4eb13bf208 100644 --- a/src/Columns/ColumnSparse.h +++ b/src/Columns/ColumnSparse.h @@ -132,6 +132,7 @@ public: void getIndicesOfNonDefaultRows(IColumn::Offsets & indices, size_t from, size_t limit) const override; double getRatioOfDefaultRows(double sample_ratio) const override; + UInt64 getNumberOfDefaultRows() const override; MutableColumns scatter(ColumnIndex num_columns, const Selector & selector) const override; @@ -150,7 +151,6 @@ public: size_t sizeOfValueIfFixed() const override { return values->sizeOfValueIfFixed() + values->sizeOfValueIfFixed(); } bool isCollationSupported() const override { return values->isCollationSupported(); } - size_t getNumberOfDefaults() const { return _size - offsets->size(); } size_t getNumberOfTrailingDefaults() const { return offsets->empty() ? _size : _size - getOffsetsData().back() - 1; diff --git a/src/Columns/ColumnString.h b/src/Columns/ColumnString.h index aa251b1fda0..08c876a803d 100644 --- a/src/Columns/ColumnString.h +++ b/src/Columns/ColumnString.h @@ -187,8 +187,8 @@ public: void updateHashFast(SipHash & hash) const override { - hash.update(reinterpret_cast(offsets.data()), size() * sizeof(offsets[0])); - hash.update(reinterpret_cast(chars.data()), size() * sizeof(chars[0])); + hash.update(reinterpret_cast(offsets.data()), offsets.size() * sizeof(offsets[0])); + hash.update(reinterpret_cast(chars.data()), chars.size() * sizeof(chars[0])); } void insertRangeFrom(const IColumn & src, size_t start, size_t length) override; @@ -273,6 +273,11 @@ public: return getRatioOfDefaultRowsImpl(sample_ratio); } + UInt64 getNumberOfDefaultRows() const override + { + return getNumberOfDefaultRowsImpl(); + } + void getIndicesOfNonDefaultRows(Offsets & indices, size_t from, size_t limit) const override { return getIndicesOfNonDefaultRowsImpl(indices, from, limit); diff --git a/src/Columns/ColumnTuple.cpp b/src/Columns/ColumnTuple.cpp index 903540c1859..369842c7281 100644 --- a/src/Columns/ColumnTuple.cpp +++ b/src/Columns/ColumnTuple.cpp @@ -565,6 +565,11 @@ double ColumnTuple::getRatioOfDefaultRows(double sample_ratio) const return getRatioOfDefaultRowsImpl(sample_ratio); } +UInt64 ColumnTuple::getNumberOfDefaultRows() const +{ + return getNumberOfDefaultRowsImpl(); +} + void ColumnTuple::getIndicesOfNonDefaultRows(Offsets & indices, size_t from, size_t limit) const { return getIndicesOfNonDefaultRowsImpl(indices, from, limit); diff --git a/src/Columns/ColumnTuple.h b/src/Columns/ColumnTuple.h index 25f6328b3fc..281fd94d893 100644 --- a/src/Columns/ColumnTuple.h +++ b/src/Columns/ColumnTuple.h @@ -102,6 +102,7 @@ public: bool isCollationSupported() const override; ColumnPtr compress() const override; double getRatioOfDefaultRows(double sample_ratio) const override; + UInt64 getNumberOfDefaultRows() const override; void getIndicesOfNonDefaultRows(Offsets & indices, size_t from, size_t limit) const override; void finalize() override; bool isFinalized() const override; diff --git a/src/Columns/ColumnUnique.h b/src/Columns/ColumnUnique.h index 8a95726d2be..377255d80c7 100644 --- a/src/Columns/ColumnUnique.h +++ b/src/Columns/ColumnUnique.h @@ -146,6 +146,11 @@ public: throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method 'getRatioOfDefaultRows' not implemented for ColumnUnique"); } + UInt64 getNumberOfDefaultRows() const override + { + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method 'getNumberOfDefaultRows' not implemented for ColumnUnique"); + } + void getIndicesOfNonDefaultRows(IColumn::Offsets &, size_t, size_t) const override { throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method 'getIndicesOfNonDefaultRows' not implemented for ColumnUnique"); diff --git a/src/Columns/ColumnVector.h b/src/Columns/ColumnVector.h index ded66430160..bf790423d1d 100644 --- a/src/Columns/ColumnVector.h +++ b/src/Columns/ColumnVector.h @@ -358,6 +358,11 @@ public: return this->template getRatioOfDefaultRowsImpl(sample_ratio); } + UInt64 getNumberOfDefaultRows() const override + { + return this->template getNumberOfDefaultRowsImpl(); + } + void getIndicesOfNonDefaultRows(IColumn::Offsets & indices, size_t from, size_t limit) const override { return this->template getIndicesOfNonDefaultRowsImpl(indices, from, limit); diff --git a/src/Columns/IColumn.h b/src/Columns/IColumn.h index 53619c73e5b..d777fd7b240 100644 --- a/src/Columns/IColumn.h +++ b/src/Columns/IColumn.h @@ -441,10 +441,13 @@ public: throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method structureEquals is not supported for {}", getName()); } - /// Returns ration of values in column, that equal to default value of column. + /// Returns ratio of values in column, that are equal to default value of column. /// Checks only @sample_ratio ratio of rows. [[nodiscard]] virtual double getRatioOfDefaultRows(double sample_ratio = 1.0) const = 0; /// NOLINT + /// Returns number of values in column, that are equal to default value of column. + [[nodiscard]] virtual UInt64 getNumberOfDefaultRows() const = 0; + /// Returns indices of values in column, that not equal to default value of column. virtual void getIndicesOfNonDefaultRows(Offsets & indices, size_t from, size_t limit) const = 0; @@ -584,6 +587,9 @@ protected: template double getRatioOfDefaultRowsImpl(double sample_ratio) const; + template + UInt64 getNumberOfDefaultRowsImpl() const; + template void getIndicesOfNonDefaultRowsImpl(Offsets & indices, size_t from, size_t limit) const; diff --git a/src/Columns/IColumnDummy.h b/src/Columns/IColumnDummy.h index 0b00ebbdd1f..82d4c857b29 100644 --- a/src/Columns/IColumnDummy.h +++ b/src/Columns/IColumnDummy.h @@ -168,6 +168,11 @@ public: throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method getRatioOfDefaultRows is not supported for {}", getName()); } + UInt64 getNumberOfDefaultRows() const override + { + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method getNumberOfDefaultRows is not supported for {}", getName()); + } + void getIndicesOfNonDefaultRows(Offsets &, size_t, size_t) const override { throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method getIndicesOfNonDefaultRows is not supported for {}", getName()); diff --git a/src/Columns/IColumnImpl.h b/src/Columns/IColumnImpl.h index 8537a4c097c..0eab9452813 100644 --- a/src/Columns/IColumnImpl.h +++ b/src/Columns/IColumnImpl.h @@ -164,16 +164,14 @@ double IColumn::getRatioOfDefaultRowsImpl(double sample_ratio) const throw Exception(ErrorCodes::LOGICAL_ERROR, "Value of 'sample_ratio' must be in interval (0.0; 1.0], but got: {}", sample_ratio); - /// Randomize a little to avoid boundary effects. - std::uniform_int_distribution dist(1, static_cast(1.0 / sample_ratio)); + static constexpr auto max_number_of_rows_for_full_search = 1000; size_t num_rows = size(); - size_t num_sampled_rows = static_cast(num_rows * sample_ratio); - size_t num_checked_rows = dist(thread_local_rng); - num_sampled_rows = std::min(num_sampled_rows + dist(thread_local_rng), num_rows); + size_t num_sampled_rows = std::min(static_cast(num_rows * sample_ratio), num_rows); + size_t num_checked_rows = 0; size_t res = 0; - if (num_sampled_rows == num_rows) + if (num_sampled_rows == num_rows || num_rows <= max_number_of_rows_for_full_search) { for (size_t i = 0; i < num_rows; ++i) res += static_cast(*this).isDefaultAt(i); @@ -181,7 +179,7 @@ double IColumn::getRatioOfDefaultRowsImpl(double sample_ratio) const } else if (num_sampled_rows != 0) { - for (size_t i = num_checked_rows; i < num_rows; ++i) + for (size_t i = 0; i < num_rows; ++i) { if (num_checked_rows * num_rows <= i * num_sampled_rows) { @@ -191,9 +189,22 @@ double IColumn::getRatioOfDefaultRowsImpl(double sample_ratio) const } } + if (num_checked_rows == 0) + return 0.0; + return static_cast(res) / num_checked_rows; } +template +UInt64 IColumn::getNumberOfDefaultRowsImpl() const +{ + UInt64 res = 0; + size_t num_rows = size(); + for (size_t i = 0; i < num_rows; ++i) + res += static_cast(*this).isDefaultAt(i); + return res; +} + template void IColumn::getIndicesOfNonDefaultRowsImpl(Offsets & indices, size_t from, size_t limit) const { diff --git a/src/Columns/tests/gtest_column_sparse.cpp b/src/Columns/tests/gtest_column_sparse.cpp index b5b226c6862..6062ea51941 100644 --- a/src/Columns/tests/gtest_column_sparse.cpp +++ b/src/Columns/tests/gtest_column_sparse.cpp @@ -327,4 +327,3 @@ TEST(ColumnSparse, GetPermutation) } #undef DUMP_COLUMN -#undef DUMP_NON_DEFAULTS diff --git a/src/Columns/tests/gtest_weak_hash_32.cpp b/src/Columns/tests/gtest_weak_hash_32.cpp index 5755cc3af72..cbf47790b9f 100644 --- a/src/Columns/tests/gtest_weak_hash_32.cpp +++ b/src/Columns/tests/gtest_weak_hash_32.cpp @@ -14,7 +14,7 @@ #include #include -#include +#include #include #include diff --git a/src/Common/BinStringDecodeHelper.h b/src/Common/BinStringDecodeHelper.h index 513a4196b6f..df3e014cfad 100644 --- a/src/Common/BinStringDecodeHelper.h +++ b/src/Common/BinStringDecodeHelper.h @@ -1,6 +1,6 @@ #pragma once -#include +#include namespace DB { diff --git a/src/Common/ConcurrentBoundedQueue.h b/src/Common/ConcurrentBoundedQueue.h index d3abf86c817..8b813911e23 100644 --- a/src/Common/ConcurrentBoundedQueue.h +++ b/src/Common/ConcurrentBoundedQueue.h @@ -1,6 +1,6 @@ #pragma once -#include +#include #include #include #include @@ -18,7 +18,8 @@ template class ConcurrentBoundedQueue { private: - std::queue queue; + using Container = std::deque; + Container queue; mutable std::mutex queue_mutex; std::condition_variable push_condition; @@ -28,7 +29,7 @@ private: size_t max_fill = 0; - template + template bool emplaceImpl(std::optional timeout_milliseconds, Args &&...args) { { @@ -51,13 +52,17 @@ private: if (is_finished) return false; - queue.emplace(std::forward(args)...); + if constexpr (back) + queue.emplace_back(std::forward(args)...); + else + queue.emplace_front(std::forward(args)...); } pop_condition.notify_one(); return true; } + template bool popImpl(T & x, std::optional timeout_milliseconds) { { @@ -80,8 +85,16 @@ private: if (is_finished && queue.empty()) return false; - detail::moveOrCopyIfThrow(std::move(queue.front()), x); - queue.pop(); + if constexpr (front) + { + detail::moveOrCopyIfThrow(std::move(queue.front()), x); + queue.pop_front(); + } + else + { + detail::moveOrCopyIfThrow(std::move(queue.back()), x); + queue.pop_back(); + } } push_condition.notify_one(); @@ -94,6 +107,12 @@ public: : max_fill(max_fill_) {} + /// Returns false if queue is finished + [[nodiscard]] bool pushFront(const T & x) + { + return emplaceImpl(/* timeout_milliseconds= */ std::nullopt , x); + } + /// Returns false if queue is finished [[nodiscard]] bool push(const T & x) { @@ -109,37 +128,37 @@ public: template [[nodiscard]] bool emplace(Args &&... args) { - return emplaceImpl(std::nullopt /* timeout in milliseconds */, std::forward(args...)); - } - - /// Returns false if queue is finished and empty - [[nodiscard]] bool pop(T & x) - { - return popImpl(x, std::nullopt /*timeout in milliseconds*/); + return emplaceImpl(std::nullopt /* timeout in milliseconds */, std::forward(args...)); } /// Returns false if queue is finished or object was not pushed during timeout [[nodiscard]] bool tryPush(const T & x, UInt64 milliseconds = 0) { - return emplaceImpl(milliseconds, x); + return emplaceImpl(milliseconds, x); } [[nodiscard]] bool tryPush(T && x, UInt64 milliseconds = 0) { - return emplaceImpl(milliseconds, std::move(x)); + return emplaceImpl(milliseconds, std::move(x)); } /// Returns false if queue is finished or object was not emplaced during timeout template [[nodiscard]] bool tryEmplace(UInt64 milliseconds, Args &&... args) { - return emplaceImpl(milliseconds, std::forward(args...)); + return emplaceImpl(milliseconds, std::forward(args...)); + } + + /// Returns false if queue is finished and empty + [[nodiscard]] bool pop(T & x) + { + return popImpl(x, std::nullopt /*timeout in milliseconds*/); } /// Returns false if queue is (finished and empty) or (object was not popped during timeout) [[nodiscard]] bool tryPop(T & x, UInt64 milliseconds) { - return popImpl(x, milliseconds); + return popImpl(x, milliseconds); } /// Returns false if queue is empty. @@ -153,7 +172,7 @@ public: return false; detail::moveOrCopyIfThrow(std::move(queue.front()), x); - queue.pop(); + queue.pop_front(); } push_condition.notify_one(); @@ -222,7 +241,7 @@ public: if (is_finished) return; - std::queue empty_queue; + Container empty_queue; queue.swap(empty_queue); } @@ -235,7 +254,7 @@ public: { std::lock_guard lock(queue_mutex); - std::queue empty_queue; + Container empty_queue; queue.swap(empty_queue); is_finished = true; } diff --git a/src/Common/CurrentMetrics.cpp b/src/Common/CurrentMetrics.cpp index 0395f2470af..5b20d98aa01 100644 --- a/src/Common/CurrentMetrics.cpp +++ b/src/Common/CurrentMetrics.cpp @@ -11,13 +11,21 @@ M(ReplicatedSend, "Number of data parts being sent to replicas") \ M(ReplicatedChecks, "Number of data parts checking for consistency") \ M(BackgroundMergesAndMutationsPoolTask, "Number of active merges and mutations in an associated background pool") \ + M(BackgroundMergesAndMutationsPoolSize, "Limit on number of active merges and mutations in an associated background pool") \ M(BackgroundFetchesPoolTask, "Number of active fetches in an associated background pool") \ + M(BackgroundFetchesPoolSize, "Limit on number of simultaneous fetches in an associated background pool") \ M(BackgroundCommonPoolTask, "Number of active tasks in an associated background pool") \ + M(BackgroundCommonPoolSize, "Limit on number of tasks in an associated background pool") \ M(BackgroundMovePoolTask, "Number of active tasks in BackgroundProcessingPool for moves") \ + M(BackgroundMovePoolSize, "Limit on number of tasks in BackgroundProcessingPool for moves") \ M(BackgroundSchedulePoolTask, "Number of active tasks in BackgroundSchedulePool. This pool is used for periodic ReplicatedMergeTree tasks, like cleaning old data parts, altering data parts, replica re-initialization, etc.") \ + M(BackgroundSchedulePoolSize, "Limit on number of tasks in BackgroundSchedulePool. This pool is used for periodic ReplicatedMergeTree tasks, like cleaning old data parts, altering data parts, replica re-initialization, etc.") \ M(BackgroundBufferFlushSchedulePoolTask, "Number of active tasks in BackgroundBufferFlushSchedulePool. This pool is used for periodic Buffer flushes") \ + M(BackgroundBufferFlushSchedulePoolSize, "Limit on number of tasks in BackgroundBufferFlushSchedulePool") \ M(BackgroundDistributedSchedulePoolTask, "Number of active tasks in BackgroundDistributedSchedulePool. This pool is used for distributed sends that is done in background.") \ + M(BackgroundDistributedSchedulePoolSize, "Limit on number of tasks in BackgroundDistributedSchedulePool") \ M(BackgroundMessageBrokerSchedulePoolTask, "Number of active tasks in BackgroundProcessingPool for message streaming") \ + M(BackgroundMessageBrokerSchedulePoolSize, "Limit on number of tasks in BackgroundProcessingPool for message streaming") \ M(CacheDictionaryUpdateQueueBatches, "Number of 'batches' (a set of keys) in update queue in CacheDictionaries.") \ M(CacheDictionaryUpdateQueueKeys, "Exact number of keys in update queue in CacheDictionaries.") \ M(DiskSpaceReservedForMerge, "Disk space reserved for currently running background merges. It is slightly more than the total size of currently merging parts.") \ @@ -84,10 +92,6 @@ M(MMappedFileBytes, "Sum size of mmapped file regions.") \ M(MMappedAllocs, "Total number of mmapped allocations") \ M(MMappedAllocBytes, "Sum bytes of mmapped allocations") \ - M(AsyncDrainedConnections, "Number of connections drained asynchronously.") \ - M(ActiveAsyncDrainedConnections, "Number of active connections drained asynchronously.") \ - M(SyncDrainedConnections, "Number of connections drained synchronously.") \ - M(ActiveSyncDrainedConnections, "Number of active connections drained synchronously.") \ M(AsynchronousReadWait, "Number of threads waiting for asynchronous read.") \ M(PendingAsyncInsert, "Number of asynchronous inserts that are waiting for flush.") \ M(KafkaConsumers, "Number of active Kafka consumers") \ diff --git a/src/Common/CurrentThread.cpp b/src/Common/CurrentThread.cpp index 526e28c043d..188e78fe69b 100644 --- a/src/Common/CurrentThread.cpp +++ b/src/Common/CurrentThread.cpp @@ -67,8 +67,8 @@ void CurrentThread::attachInternalTextLogsQueue(const std::shared_ptr callback) { - if (unlikely(!current_thread)) - return; + /// It does not make sense to set a callback for sending logs to a client if there's no thread status + chassert(current_thread); current_thread->setFatalErrorCallback(callback); } @@ -110,27 +110,4 @@ ThreadGroupStatusPtr CurrentThread::getGroup() return current_thread->getThreadGroup(); } -MemoryTracker * CurrentThread::getUserMemoryTracker() -{ - if (unlikely(!current_thread)) - return nullptr; - - auto * tracker = current_thread->memory_tracker.getParent(); - while (tracker && tracker->level != VariableContext::User) - tracker = tracker->getParent(); - - return tracker; -} - -void CurrentThread::flushUntrackedMemory() -{ - if (unlikely(!current_thread)) - return; - if (current_thread->untracked_memory == 0) - return; - - current_thread->memory_tracker.adjustWithUntrackedMemory(current_thread->untracked_memory); - current_thread->untracked_memory = 0; -} - } diff --git a/src/Common/CurrentThread.h b/src/Common/CurrentThread.h index f36b92e319d..f4975e800ca 100644 --- a/src/Common/CurrentThread.h +++ b/src/Common/CurrentThread.h @@ -40,12 +40,6 @@ public: /// Group to which belongs current thread static ThreadGroupStatusPtr getGroup(); - /// MemoryTracker for user that owns current thread if any - static MemoryTracker * getUserMemoryTracker(); - - /// Adjust counters in MemoryTracker hierarchy if untracked_memory is not 0. - static void flushUntrackedMemory(); - /// A logs queue used by TCPHandler to pass logs to a client static void attachInternalTextLogsQueue(const std::shared_ptr & logs_queue, LogsLevel client_logs_level); @@ -100,8 +94,8 @@ public: /// Initializes query with current thread as master thread in constructor, and detaches it in destructor struct QueryScope : private boost::noncopyable { - explicit QueryScope(ContextMutablePtr query_context); - explicit QueryScope(ContextPtr query_context); + explicit QueryScope(ContextMutablePtr query_context, std::function fatal_error_callback = {}); + explicit QueryScope(ContextPtr query_context, std::function fatal_error_callback = {}); ~QueryScope(); void logPeakMemoryUsage(); diff --git a/src/Common/LRUCachePolicy.h b/src/Common/LRUCachePolicy.h index 3c069eb276b..b6c0ef0d3ef 100644 --- a/src/Common/LRUCachePolicy.h +++ b/src/Common/LRUCachePolicy.h @@ -31,7 +31,7 @@ public: * max_elements_size == 0 means no elements size restrictions. */ explicit LRUCachePolicy(size_t max_size_, size_t max_elements_size_ = 0, OnWeightLossFunction on_weight_loss_function_ = {}) - : max_size(std::max(static_cast(1), max_size_)), max_elements_size(max_elements_size_) + : max_size(std::max(1uz, max_size_)), max_elements_size(max_elements_size_) { Base::on_weight_loss_function = on_weight_loss_function_; } diff --git a/src/Common/LoggingFormatStringHelpers.cpp b/src/Common/LoggingFormatStringHelpers.cpp index 6b2f32c06e3..85659e45791 100644 --- a/src/Common/LoggingFormatStringHelpers.cpp +++ b/src/Common/LoggingFormatStringHelpers.cpp @@ -1,7 +1,76 @@ #include +#include +#include [[noreturn]] void functionThatFailsCompilationOfConstevalFunctions(const char * error) { throw std::runtime_error(error); } +std::unordered_map> LogFrequencyLimiterIml::logged_messages; +time_t LogFrequencyLimiterIml::last_cleanup = 0; +std::mutex LogFrequencyLimiterIml::mutex; + +void LogFrequencyLimiterIml::log(Poco::Message & message) +{ + std::string_view pattern = message.getFormatString(); + if (pattern.empty()) + { + /// Do not filter messages without a format string + if (auto * channel = logger->getChannel()) + channel->log(message); + return; + } + + SipHash hash; + hash.update(logger->name()); + /// Format strings are compile-time constants, so they are uniquely identified by pointer and size + hash.update(pattern.data()); + hash.update(pattern.size()); + + time_t now = time(nullptr); + size_t skipped_similar_messages = 0; + bool need_cleanup; + bool need_log; + + { + std::lock_guard lock(mutex); + need_cleanup = last_cleanup + 300 <= now; + auto & info = logged_messages[hash.get64()]; + need_log = info.first + min_interval_s <= now; + if (need_log) + { + skipped_similar_messages = info.second; + info.first = now; + info.second = 0; + } + else + { + ++info.second; + } + } + + /// We don't need all threads to do cleanup, just randomize + if (need_cleanup && thread_local_rng() % 100 == 0) + cleanup(); + + /// The message it too frequent, skip it for now + /// NOTE It's not optimal because we format the message first and only then check if we need to actually write it, see LOG_IMPL macro + if (!need_log) + return; + + if (skipped_similar_messages) + message.appendText(fmt::format(" (skipped {} similar messages)", skipped_similar_messages)); + + if (auto * channel = logger->getChannel()) + channel->log(message); +} + +void LogFrequencyLimiterIml::cleanup(time_t too_old_threshold_s) +{ + time_t now = time(nullptr); + time_t old = now - too_old_threshold_s; + std::lock_guard lock(mutex); + std::erase_if(logged_messages, [old](const auto & elem) { return elem.second.first < old; }); + last_cleanup = now; +} diff --git a/src/Common/LoggingFormatStringHelpers.h b/src/Common/LoggingFormatStringHelpers.h index 2aed614bd8b..b29510a2c93 100644 --- a/src/Common/LoggingFormatStringHelpers.h +++ b/src/Common/LoggingFormatStringHelpers.h @@ -1,6 +1,11 @@ #pragma once #include +#include #include +#include +#include +#include +#include struct PreformattedMessage; consteval void formatStringCheckArgsNumImpl(std::string_view str, size_t nargs); @@ -156,3 +161,59 @@ struct CheckArgsNumHelperImpl template using CheckArgsNumHelper = CheckArgsNumHelperImpl...>; template void formatStringCheckArgsNum(CheckArgsNumHelper, Args &&...) {} + + +/// This wrapper helps to avoid too frequent and noisy log messages. +/// For each pair (logger_name, format_string) it remembers when such a message was logged the last time. +/// The message will not be logged again if less than min_interval_s seconds passed since the previously logged message. +class LogFrequencyLimiterIml +{ + /// Hash(logger_name, format_string) -> (last_logged_time_s, skipped_messages_count) + static std::unordered_map> logged_messages; + static time_t last_cleanup; + static std::mutex mutex; + + Poco::Logger * logger; + time_t min_interval_s; +public: + LogFrequencyLimiterIml(Poco::Logger * logger_, time_t min_interval_s_) : logger(logger_), min_interval_s(min_interval_s_) {} + + LogFrequencyLimiterIml & operator -> () { return *this; } + bool is(Poco::Message::Priority priority) { return logger->is(priority); } + LogFrequencyLimiterIml * getChannel() {return this; } + const String & name() const { return logger->name(); } + + void log(Poco::Message & message); + + /// Clears messages that were logged last time more than too_old_threshold_s seconds ago + static void cleanup(time_t too_old_threshold_s = 600); + + Poco::Logger * getLogger() { return logger; } +}; + +/// This wrapper is useful to save formatted message into a String before sending it to a logger +class LogToStrImpl +{ + String & out_str; + Poco::Logger * logger; + std::unique_ptr maybe_nested; + bool propagate_to_actual_log = true; +public: + LogToStrImpl(String & out_str_, Poco::Logger * logger_) : out_str(out_str_), logger(logger_) {} + LogToStrImpl(String & out_str_, std::unique_ptr && maybe_nested_) + : out_str(out_str_), logger(maybe_nested_->getLogger()), maybe_nested(std::move(maybe_nested_)) {} + LogToStrImpl & operator -> () { return *this; } + bool is(Poco::Message::Priority priority) { propagate_to_actual_log &= logger->is(priority); return true; } + LogToStrImpl * getChannel() {return this; } + const String & name() const { return logger->name(); } + void log(Poco::Message & message) + { + out_str = message.getText(); + if (!propagate_to_actual_log) + return; + if (maybe_nested) + maybe_nested->log(message); + else if (auto * channel = logger->getChannel()) + channel->log(message); + } +}; diff --git a/src/Common/OpenTelemetryTraceContext.cpp b/src/Common/OpenTelemetryTraceContext.cpp index b62822ceda2..014034c6ba8 100644 --- a/src/Common/OpenTelemetryTraceContext.cpp +++ b/src/Common/OpenTelemetryTraceContext.cpp @@ -3,7 +3,7 @@ #include #include #include -#include +#include #include #include @@ -92,7 +92,7 @@ bool Span::addAttributeImpl(std::string_view name, std::string_view value) noexc return true; } -SpanHolder::SpanHolder(std::string_view _operation_name) +SpanHolder::SpanHolder(std::string_view _operation_name, SpanKind _kind) { if (!current_thread_trace_context.isTraceEnabled()) { @@ -106,6 +106,7 @@ SpanHolder::SpanHolder(std::string_view _operation_name) this->parent_span_id = current_thread_trace_context.span_id; this->span_id = thread_local_rng(); // create a new id for this span this->operation_name = _operation_name; + this->kind = _kind; this->start_time_us = std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count(); diff --git a/src/Common/OpenTelemetryTraceContext.h b/src/Common/OpenTelemetryTraceContext.h index 64bce9e98db..bfd091e2902 100644 --- a/src/Common/OpenTelemetryTraceContext.h +++ b/src/Common/OpenTelemetryTraceContext.h @@ -13,6 +13,29 @@ class ReadBuffer; namespace OpenTelemetry { +/// See https://opentelemetry.io/docs/reference/specification/trace/api/#spankind +enum SpanKind +{ + /// Default value. Indicates that the span represents an internal operation within an application, + /// as opposed to an operations with remote parents or children. + INTERNAL = 0, + + /// Indicates that the span covers server-side handling of a synchronous RPC or other remote request. + /// This span is often the child of a remote CLIENT span that was expected to wait for a response. + SERVER = 1, + + /// Indicates that the span describes a request to some remote service. + /// This span is usually the parent of a remote SERVER span and does not end until the response is received. + CLIENT = 2, + + /// Indicates that the span describes the initiators of an asynchronous request. This parent span will often end before the corresponding child CONSUMER span, possibly even before the child span starts. + /// In messaging scenarios with batching, tracing individual messages requires a new PRODUCER span per message to be created. + PRODUCER = 3, + + /// Indicates that the span describes a child of an asynchronous PRODUCER request + CONSUMER = 4 +}; + struct Span { UUID trace_id{}; @@ -21,6 +44,7 @@ struct Span String operation_name; UInt64 start_time_us = 0; UInt64 finish_time_us = 0; + SpanKind kind = INTERNAL; Map attributes; /// Following methods are declared as noexcept to make sure they're exception safe. @@ -155,7 +179,7 @@ using TracingContextHolderPtr = std::unique_ptr; /// Once it's created or destructed, it automatically maitains the tracing context on the thread that it lives. struct SpanHolder : public Span { - SpanHolder(std::string_view); + SpanHolder(std::string_view, SpanKind _kind = INTERNAL); ~SpanHolder(); /// Finish a span explicitly if needed. diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp index 1b3ce409a00..a031221a725 100644 --- a/src/Common/ProfileEvents.cpp +++ b/src/Common/ProfileEvents.cpp @@ -166,6 +166,8 @@ \ M(WaitMarksLoadMicroseconds, "Time spent loading marks") \ M(BackgroundLoadingMarksTasks, "Number of background tasks for loading marks") \ + M(LoadedMarksCount, "Number of marks loaded (total across columns).") \ + M(LoadedMarksMemoryBytes, "Size of in-memory representations of loaded marks.") \ \ M(Merge, "Number of launched background merges.") \ M(MergedRows, "Rows read for background merges. This is the number of rows before merge.") \ diff --git a/src/Common/RWLock.cpp b/src/Common/RWLock.cpp index c2419d0c1b7..2d0fcfa3e74 100644 --- a/src/Common/RWLock.cpp +++ b/src/Common/RWLock.cpp @@ -97,7 +97,7 @@ private: * Note: "SM" in the commentaries below stands for STATE MODIFICATION */ RWLockImpl::LockHolder -RWLockImpl::getLock(RWLockImpl::Type type, const String & query_id, const std::chrono::milliseconds & lock_timeout_ms) +RWLockImpl::getLock(RWLockImpl::Type type, const String & query_id, const std::chrono::milliseconds & lock_timeout_ms, bool throw_in_fast_path) { const auto lock_deadline_tp = (lock_timeout_ms == std::chrono::milliseconds(0)) @@ -130,11 +130,19 @@ RWLockImpl::getLock(RWLockImpl::Type type, const String & query_id, const std::c if (owner_query_it != owner_queries.end()) { if (wrlock_owner != writers_queue.end()) - throw Exception(ErrorCodes::LOGICAL_ERROR, "RWLockImpl::getLock(): RWLock is already locked in exclusive mode"); + { + if (throw_in_fast_path) + throw Exception(ErrorCodes::LOGICAL_ERROR, "RWLockImpl::getLock(): RWLock is already locked in exclusive mode"); + return nullptr; + } /// Lock upgrading is not supported if (type == Write) - throw Exception(ErrorCodes::LOGICAL_ERROR, "RWLockImpl::getLock(): Cannot acquire exclusive lock while RWLock is already locked"); + { + if (throw_in_fast_path) + throw Exception(ErrorCodes::LOGICAL_ERROR, "RWLockImpl::getLock(): Cannot acquire exclusive lock while RWLock is already locked"); + return nullptr; + } /// N.B. Type is Read here, query_id is not empty and it_query is a valid iterator ++owner_query_it->second; /// SM1: nothrow diff --git a/src/Common/RWLock.h b/src/Common/RWLock.h index cb4cf7f9200..dd965b65026 100644 --- a/src/Common/RWLock.h +++ b/src/Common/RWLock.h @@ -56,7 +56,7 @@ public: /// Empty query_id means the lock is acquired from outside of query context (e.g. in a background thread). LockHolder getLock(Type type, const String & query_id, - const std::chrono::milliseconds & lock_timeout_ms = std::chrono::milliseconds(0)); + const std::chrono::milliseconds & lock_timeout_ms = std::chrono::milliseconds(0), bool throw_in_fast_path = true); /// Use as query_id to acquire a lock outside the query context. inline static const String NO_QUERY = String(); diff --git a/src/Common/StackTrace.cpp b/src/Common/StackTrace.cpp index 0d47d3dcb92..3c0bcd2f808 100644 --- a/src/Common/StackTrace.cpp +++ b/src/Common/StackTrace.cpp @@ -1,18 +1,21 @@ -#include +#include "StackTrace.h" + +#include +#include +#include #include #include -#include #include -#include +#include #include -#include #include +#include #include #include #include -#include +#include #include "config.h" @@ -20,24 +23,23 @@ # include #endif - namespace { - /// Currently this variable is set up once on server startup. - /// But we use atomic just in case, so it is possible to be modified at runtime. - std::atomic show_addresses = true; +/// Currently this variable is set up once on server startup. +/// But we use atomic just in case, so it is possible to be modified at runtime. +std::atomic show_addresses = true; - bool shouldShowAddress(const void * addr) - { - /// If the address is less than 4096, most likely it is a nullptr dereference with offset, - /// and showing this offset is secure nevertheless. - /// NOTE: 4096 is the page size on x86 and it can be different on other systems, - /// but for the purpose of this branch, it does not matter. - if (reinterpret_cast(addr) < 4096) - return true; +bool shouldShowAddress(const void * addr) +{ + /// If the address is less than 4096, most likely it is a nullptr dereference with offset, + /// and showing this offset is secure nevertheless. + /// NOTE: 4096 is the page size on x86 and it can be different on other systems, + /// but for the purpose of this branch, it does not matter. + if (reinterpret_cast(addr) < 4096) + return true; - return show_addresses.load(std::memory_order_relaxed); - } + return show_addresses.load(std::memory_order_relaxed); +} } void StackTrace::setShowAddresses(bool show) @@ -45,155 +47,129 @@ void StackTrace::setShowAddresses(bool show) show_addresses.store(show, std::memory_order_relaxed); } +std::string SigsegvErrorString(const siginfo_t & info, [[maybe_unused]] const ucontext_t & context) +{ + using namespace std::string_literals; + std::string address + = info.si_addr == nullptr ? "NULL pointer"s : (shouldShowAddress(info.si_addr) ? fmt::format("{}", info.si_addr) : ""s); + + const std::string_view access = +#if defined(__x86_64__) && !defined(OS_FREEBSD) && !defined(OS_DARWIN) && !defined(__arm__) && !defined(__powerpc__) + (context.uc_mcontext.gregs[REG_ERR] & 0x02) ? "write" : "read"; +#else + ""; +#endif + + std::string_view message; + + switch (info.si_code) + { + case SEGV_ACCERR: + message = "Attempted access has violated the permissions assigned to the memory area"; + break; + case SEGV_MAPERR: + message = "Address not mapped to object"; + break; + default: + message = "Unknown si_code"; + break; + } + + return fmt::format("Address: {}. Access: {}. {}.", std::move(address), access, message); +} + +constexpr std::string_view SigbusErrorString(int si_code) +{ + switch (si_code) + { + case BUS_ADRALN: + return "Invalid address alignment."; + case BUS_ADRERR: + return "Non-existent physical address."; + case BUS_OBJERR: + return "Object specific hardware error."; + + // Linux specific +#if defined(BUS_MCEERR_AR) + case BUS_MCEERR_AR: + return "Hardware memory error: action required."; +#endif +#if defined(BUS_MCEERR_AO) + case BUS_MCEERR_AO: + return "Hardware memory error: action optional."; +#endif + default: + return "Unknown si_code."; + } +} + +constexpr std::string_view SigfpeErrorString(int si_code) +{ + switch (si_code) + { + case FPE_INTDIV: + return "Integer divide by zero."; + case FPE_INTOVF: + return "Integer overflow."; + case FPE_FLTDIV: + return "Floating point divide by zero."; + case FPE_FLTOVF: + return "Floating point overflow."; + case FPE_FLTUND: + return "Floating point underflow."; + case FPE_FLTRES: + return "Floating point inexact result."; + case FPE_FLTINV: + return "Floating point invalid operation."; + case FPE_FLTSUB: + return "Subscript out of range."; + default: + return "Unknown si_code."; + } +} + +constexpr std::string_view SigillErrorString(int si_code) +{ + switch (si_code) + { + case ILL_ILLOPC: + return "Illegal opcode."; + case ILL_ILLOPN: + return "Illegal operand."; + case ILL_ILLADR: + return "Illegal addressing mode."; + case ILL_ILLTRP: + return "Illegal trap."; + case ILL_PRVOPC: + return "Privileged opcode."; + case ILL_PRVREG: + return "Privileged register."; + case ILL_COPROC: + return "Coprocessor error."; + case ILL_BADSTK: + return "Internal stack error."; + default: + return "Unknown si_code."; + } +} std::string signalToErrorMessage(int sig, const siginfo_t & info, [[maybe_unused]] const ucontext_t & context) { - std::stringstream error; // STYLE_CHECK_ALLOW_STD_STRING_STREAM - error.exceptions(std::ios::failbit); switch (sig) { case SIGSEGV: - { - /// Print info about address and reason. - if (nullptr == info.si_addr) - error << "Address: NULL pointer."; - else if (shouldShowAddress(info.si_addr)) - error << "Address: " << info.si_addr; - -#if defined(__x86_64__) && !defined(OS_FREEBSD) && !defined(OS_DARWIN) && !defined(__arm__) && !defined(__powerpc__) - auto err_mask = context.uc_mcontext.gregs[REG_ERR]; - if ((err_mask & 0x02)) - error << " Access: write."; - else - error << " Access: read."; -#endif - - switch (info.si_code) - { - case SEGV_ACCERR: - error << " Attempted access has violated the permissions assigned to the memory area."; - break; - case SEGV_MAPERR: - error << " Address not mapped to object."; - break; - default: - error << " Unknown si_code."; - break; - } - break; - } - + return SigsegvErrorString(info, context); case SIGBUS: - { - switch (info.si_code) - { - case BUS_ADRALN: - error << "Invalid address alignment."; - break; - case BUS_ADRERR: - error << "Non-existent physical address."; - break; - case BUS_OBJERR: - error << "Object specific hardware error."; - break; - - // Linux specific -#if defined(BUS_MCEERR_AR) - case BUS_MCEERR_AR: - error << "Hardware memory error: action required."; - break; -#endif -#if defined(BUS_MCEERR_AO) - case BUS_MCEERR_AO: - error << "Hardware memory error: action optional."; - break; -#endif - - default: - error << "Unknown si_code."; - break; - } - break; - } - + return std::string{SigbusErrorString(info.si_code)}; case SIGILL: - { - switch (info.si_code) - { - case ILL_ILLOPC: - error << "Illegal opcode."; - break; - case ILL_ILLOPN: - error << "Illegal operand."; - break; - case ILL_ILLADR: - error << "Illegal addressing mode."; - break; - case ILL_ILLTRP: - error << "Illegal trap."; - break; - case ILL_PRVOPC: - error << "Privileged opcode."; - break; - case ILL_PRVREG: - error << "Privileged register."; - break; - case ILL_COPROC: - error << "Coprocessor error."; - break; - case ILL_BADSTK: - error << "Internal stack error."; - break; - default: - error << "Unknown si_code."; - break; - } - break; - } - + return std::string{SigillErrorString(info.si_code)}; case SIGFPE: - { - switch (info.si_code) - { - case FPE_INTDIV: - error << "Integer divide by zero."; - break; - case FPE_INTOVF: - error << "Integer overflow."; - break; - case FPE_FLTDIV: - error << "Floating point divide by zero."; - break; - case FPE_FLTOVF: - error << "Floating point overflow."; - break; - case FPE_FLTUND: - error << "Floating point underflow."; - break; - case FPE_FLTRES: - error << "Floating point inexact result."; - break; - case FPE_FLTINV: - error << "Floating point invalid operation."; - break; - case FPE_FLTSUB: - error << "Subscript out of range."; - break; - default: - error << "Unknown si_code."; - break; - } - break; - } - + return std::string{SigfpeErrorString(info.si_code)}; case SIGTSTP: - { - error << "This is a signal used for debugging purposes by the user."; - break; - } + return "This is a signal used for debugging purposes by the user."; + default: + return ""; } - - return error.str(); } static void * getCallerAddress(const ucontext_t & context) @@ -207,10 +183,8 @@ static void * getCallerAddress(const ucontext_t & context) # else return reinterpret_cast(context.uc_mcontext.gregs[REG_RIP]); # endif - #elif defined(OS_DARWIN) && defined(__aarch64__) return reinterpret_cast(context.uc_mcontext->__ss.__pc); - #elif defined(OS_FREEBSD) && defined(__aarch64__) return reinterpret_cast(context.uc_mcontext.mc_gpregs.gp_elr); #elif defined(__aarch64__) @@ -228,20 +202,17 @@ static void * getCallerAddress(const ucontext_t & context) #endif } +// FIXME: looks like this is used only for Sentry but duplicates the whole algo, maybe replace? void StackTrace::symbolize( - const StackTrace::FramePointers & frame_pointers, [[maybe_unused]] size_t offset, - size_t size, StackTrace::Frames & frames) + const StackTrace::FramePointers & frame_pointers, [[maybe_unused]] size_t offset, size_t size, StackTrace::Frames & frames) { #if defined(__ELF__) && !defined(OS_FREEBSD) - auto symbol_index_ptr = DB::SymbolIndex::instance(); const DB::SymbolIndex & symbol_index = *symbol_index_ptr; std::unordered_map dwarfs; for (size_t i = 0; i < offset; ++i) - { frames[i].virtual_addr = frame_pointers[i]; - } for (size_t i = offset; i < size; ++i) { @@ -254,7 +225,7 @@ void StackTrace::symbolize( if (object) { current_frame.object = object->name; - if (std::filesystem::exists(current_frame.object.value())) + if (std::error_code ec; std::filesystem::exists(current_frame.object.value(), ec) && !ec) { auto dwarf_it = dwarfs.try_emplace(object->name, object->elf).first; @@ -269,34 +240,19 @@ void StackTrace::symbolize( } } else - { current_frame.object = "?"; - } - const auto * symbol = symbol_index.findSymbol(current_frame.virtual_addr); - if (symbol) - { - int status = 0; - current_frame.symbol = demangle(symbol->name, status); - } + if (const auto * symbol = symbol_index.findSymbol(current_frame.virtual_addr)) + current_frame.symbol = demangle(symbol->name); else - { current_frame.symbol = "?"; - } } #else for (size_t i = 0; i < size; ++i) - { frames[i].virtual_addr = frame_pointers[i]; - } #endif } -StackTrace::StackTrace() -{ - tryCapture(); -} - StackTrace::StackTrace(const ucontext_t & signal_context) { tryCapture(); @@ -325,81 +281,97 @@ StackTrace::StackTrace(const ucontext_t & signal_context) } } -StackTrace::StackTrace(NoCapture) -{ -} - void StackTrace::tryCapture() { - size = 0; #if USE_UNWIND size = unw_backtrace(frame_pointers.data(), capacity); __msan_unpoison(frame_pointers.data(), size * sizeof(frame_pointers[0])); +#else + size = 0; #endif } -size_t StackTrace::getSize() const +/// ClickHouse uses bundled libc++ so type names will be the same on every system thus it's safe to hardcode them +constexpr std::pair replacements[] + = {{"::__1", ""}, {"std::basic_string, std::allocator>", "String"}}; + +String collapseNames(String && haystack) { - return size; + // TODO: surely there is a written version already for better in place search&replace + for (auto [needle, to] : replacements) + { + size_t pos = 0; + while ((pos = haystack.find(needle, pos)) != std::string::npos) + { + haystack.replace(pos, needle.length(), to); + pos += to.length(); + } + } + + return haystack; } -size_t StackTrace::getOffset() const +struct StackTraceRefTriple { - return offset; + const StackTrace::FramePointers & pointers; + size_t offset; + size_t size; +}; + +struct StackTraceTriple +{ + StackTrace::FramePointers pointers; + size_t offset; + size_t size; +}; + +template +concept MaybeRef = std::is_same_v || std::is_same_v; + +constexpr bool operator<(const MaybeRef auto & left, const MaybeRef auto & right) +{ + return std::tuple{left.pointers, left.size, left.offset} < std::tuple{right.pointers, right.size, right.offset}; } -const StackTrace::FramePointers & StackTrace::getFramePointers() const +static void +toStringEveryLineImpl([[maybe_unused]] bool fatal, const StackTraceRefTriple & stack_trace, Fn auto && callback) { - return frame_pointers; -} - -static void toStringEveryLineImpl( - [[maybe_unused]] bool fatal, - const StackTrace::FramePointers & frame_pointers, - size_t offset, - size_t size, - std::function callback) -{ - if (size == 0) + if (stack_trace.size == 0) return callback(""); #if defined(__ELF__) && !defined(OS_FREEBSD) - auto symbol_index_ptr = DB::SymbolIndex::instance(); - const DB::SymbolIndex & symbol_index = *symbol_index_ptr; - std::unordered_map dwarfs; - - std::stringstream out; // STYLE_CHECK_ALLOW_STD_STRING_STREAM + std::stringstream out; // STYLE_CHECK_ALLOW_STD_STRING_STREAM out.exceptions(std::ios::failbit); - for (size_t i = offset; i < size; ++i) + using enum DB::Dwarf::LocationInfoMode; + const auto mode = fatal ? FULL_WITH_INLINE : FAST; + + auto symbol_index_ptr = DB::SymbolIndex::instance(); + const DB::SymbolIndex & symbol_index = *symbol_index_ptr; + std::unordered_map dwarfs; + + for (size_t i = stack_trace.offset; i < stack_trace.size; ++i) { std::vector inline_frames; - const void * virtual_addr = frame_pointers[i]; + const void * virtual_addr = stack_trace.pointers[i]; const auto * object = symbol_index.findObject(virtual_addr); uintptr_t virtual_offset = object ? uintptr_t(object->address_begin) : 0; const void * physical_addr = reinterpret_cast(uintptr_t(virtual_addr) - virtual_offset); out << i << ". "; - if (object) + if (std::error_code ec; object && std::filesystem::exists(object->name, ec) && !ec) { - if (std::filesystem::exists(object->name)) - { - auto dwarf_it = dwarfs.try_emplace(object->name, object->elf).first; + auto dwarf_it = dwarfs.try_emplace(object->name, object->elf).first; - DB::Dwarf::LocationInfo location; - auto mode = fatal ? DB::Dwarf::LocationInfoMode::FULL_WITH_INLINE : DB::Dwarf::LocationInfoMode::FAST; - if (dwarf_it->second.findAddress(uintptr_t(physical_addr), location, mode, inline_frames)) - out << location.file.toString() << ":" << location.line << ": "; - } + DB::Dwarf::LocationInfo location; + + if (dwarf_it->second.findAddress(uintptr_t(physical_addr), location, mode, inline_frames)) + out << location.file.toString() << ":" << location.line << ": "; } - const auto * symbol = symbol_index.findSymbol(virtual_addr); - if (symbol) - { - int status = 0; - out << demangle(symbol->name, status); - } + if (const auto * const symbol = symbol_index.findSymbol(virtual_addr)) + out << collapseNames(demangle(symbol->name)); else out << "?"; @@ -411,64 +383,31 @@ static void toStringEveryLineImpl( for (size_t j = 0; j < inline_frames.size(); ++j) { const auto & frame = inline_frames[j]; - int status = 0; - callback(fmt::format("{}.{}. inlined from {}:{}: {}", - i, j+1, frame.location.file.toString(), frame.location.line, demangle(frame.name, status))); + callback(fmt::format( + "{}.{}. inlined from {}:{}: {}", + i, + j + 1, + frame.location.file.toString(), + frame.location.line, + collapseNames(demangle(frame.name)))); } callback(out.str()); out.str({}); } #else - std::stringstream out; // STYLE_CHECK_ALLOW_STD_STRING_STREAM - out.exceptions(std::ios::failbit); - - for (size_t i = offset; i < size; ++i) - { - const void * addr = frame_pointers[i]; - if (shouldShowAddress(addr)) - { - out << i << ". " << addr; - - callback(out.str()); - out.str({}); - } - } + for (size_t i = stack_trace.offset; i < stack_trace.size; ++i) + if (const void * const addr = stack_trace.pointers[i]; shouldShowAddress(addr)) + callback(fmt::format("{}. {}", i, addr)); #endif } -static std::string toStringImpl(const StackTrace::FramePointers & frame_pointers, size_t offset, size_t size) +void StackTrace::toStringEveryLine(std::function callback) const { - std::stringstream out; // STYLE_CHECK_ALLOW_STD_STRING_STREAM - out.exceptions(std::ios::failbit); - toStringEveryLineImpl(false, frame_pointers, offset, size, [&](const std::string & str) { out << str << '\n'; }); - return out.str(); + toStringEveryLineImpl(true, {frame_pointers, offset, size}, std::move(callback)); } -void StackTrace::toStringEveryLine(std::function callback) const -{ - toStringEveryLineImpl(true, frame_pointers, offset, size, std::move(callback)); -} - - -std::string StackTrace::toString() const -{ - return toStringStatic(frame_pointers, offset, size); -} - -std::string StackTrace::toString(void ** frame_pointers_, size_t offset, size_t size) -{ - __msan_unpoison(frame_pointers_, size * sizeof(*frame_pointers_)); - - StackTrace::FramePointers frame_pointers_copy{}; - for (size_t i = 0; i < size; ++i) - frame_pointers_copy[i] = frame_pointers_[i]; - - return toStringStatic(frame_pointers_copy, offset, size); -} - -using StackTraceRepresentation = std::tuple; -using StackTraceCache = std::map; +using StackTraceCache = std::map>; static StackTraceCache & cacheInstance() { @@ -478,21 +417,41 @@ static StackTraceCache & cacheInstance() static std::mutex stacktrace_cache_mutex; -std::string StackTrace::toStringStatic(const StackTrace::FramePointers & frame_pointers, size_t offset, size_t size) +String toStringCached(const StackTrace::FramePointers & pointers, size_t offset, size_t size) { /// Calculation of stack trace text is extremely slow. /// We use simple cache because otherwise the server could be overloaded by trash queries. /// Note that this cache can grow unconditionally, but practically it should be small. std::lock_guard lock{stacktrace_cache_mutex}; - StackTraceRepresentation key{frame_pointers, offset, size}; - auto & cache = cacheInstance(); - if (cache.contains(key)) - return cache[key]; + StackTraceCache & cache = cacheInstance(); + const StackTraceRefTriple key{pointers, offset, size}; - auto result = toStringImpl(frame_pointers, offset, size); - cache[key] = result; - return result; + if (auto it = cache.find(key); it != cache.end()) + return it->second; + else + { + std::ostringstream out; // STYLE_CHECK_ALLOW_STD_STRING_STREAM + out.exceptions(std::ios::failbit); + toStringEveryLineImpl(false, key, [&](std::string_view str) { out << str << '\n'; }); + + return cache.emplace(StackTraceTriple{pointers, offset, size}, out.str()).first->second; + } +} + +std::string StackTrace::toString() const +{ + return toStringCached(frame_pointers, offset, size); +} + +std::string StackTrace::toString(void ** frame_pointers_raw, size_t offset, size_t size) +{ + __msan_unpoison(frame_pointers_raw, size * sizeof(*frame_pointers_raw)); + + StackTrace::FramePointers frame_pointers{}; + std::copy_n(frame_pointers_raw, size, frame_pointers.begin()); + + return toStringCached(frame_pointers, offset, size); } void StackTrace::dropCache() diff --git a/src/Common/StackTrace.h b/src/Common/StackTrace.h index f07c05107ee..3940c880c5b 100644 --- a/src/Common/StackTrace.h +++ b/src/Common/StackTrace.h @@ -46,26 +46,25 @@ public: using Frames = std::array; /// Tries to capture stack trace - StackTrace(); + inline StackTrace() { tryCapture(); } /// Tries to capture stack trace. Fallbacks on parsing caller address from /// signal context if no stack trace could be captured explicit StackTrace(const ucontext_t & signal_context); /// Creates empty object for deferred initialization - explicit StackTrace(NoCapture); + explicit inline StackTrace(NoCapture) {} - size_t getSize() const; - size_t getOffset() const; - const FramePointers & getFramePointers() const; + constexpr size_t getSize() const { return size; } + constexpr size_t getOffset() const { return offset; } + const FramePointers & getFramePointers() const { return frame_pointers; } std::string toString() const; static std::string toString(void ** frame_pointers, size_t offset, size_t size); - static std::string toStringStatic(const FramePointers & frame_pointers, size_t offset, size_t size); static void dropCache(); static void symbolize(const FramePointers & frame_pointers, size_t offset, size_t size, StackTrace::Frames & frames); - void toStringEveryLine(std::function callback) const; + void toStringEveryLine(std::function callback) const; /// Displaying the addresses can be disabled for security reasons. /// If you turn off addresses, it will be more secure, but we will be unable to help you with debugging. diff --git a/src/Common/StatusFile.cpp b/src/Common/StatusFile.cpp index d90d50ff96d..a9ffce7ddf8 100644 --- a/src/Common/StatusFile.cpp +++ b/src/Common/StatusFile.cpp @@ -51,7 +51,7 @@ StatusFile::StatusFile(std::string path_, FillFunction fill_) std::string contents; { ReadBufferFromFile in(path, 1024); - LimitReadBuffer limit_in(in, 1024, false); + LimitReadBuffer limit_in(in, 1024, /* trow_exception */ false, /* exact_limit */ {}); readStringUntilEOF(contents, limit_in); } diff --git a/src/Common/SymbolIndex.cpp b/src/Common/SymbolIndex.cpp index 6f31009b1d2..f1cace5017c 100644 --- a/src/Common/SymbolIndex.cpp +++ b/src/Common/SymbolIndex.cpp @@ -1,7 +1,7 @@ #if defined(__ELF__) && !defined(OS_FREEBSD) #include -#include +#include #include #include diff --git a/src/Common/ThreadStatus.cpp b/src/Common/ThreadStatus.cpp index 46c171b5cb6..b5a48c48ffe 100644 --- a/src/Common/ThreadStatus.cpp +++ b/src/Common/ThreadStatus.cpp @@ -144,9 +144,18 @@ ThreadStatus::ThreadStatus() #endif } +void ThreadStatus::flushUntrackedMemory() +{ + if (untracked_memory == 0) + return; + + memory_tracker.adjustWithUntrackedMemory(untracked_memory); + untracked_memory = 0; +} + ThreadStatus::~ThreadStatus() { - memory_tracker.adjustWithUntrackedMemory(untracked_memory); + flushUntrackedMemory(); if (thread_group) { @@ -226,17 +235,19 @@ void ThreadStatus::attachInternalProfileEventsQueue(const InternalProfileEventsQ void ThreadStatus::setFatalErrorCallback(std::function callback) { - fatal_error_callback = std::move(callback); - - if (!thread_group) - return; - + /// It does not make sense to set a callback for sending logs to a client if there's no thread group + chassert(thread_group); std::lock_guard lock(thread_group->mutex); + fatal_error_callback = std::move(callback); thread_group->fatal_error_callback = fatal_error_callback; } void ThreadStatus::onFatalError() { + /// No thread group - no callback + if (!thread_group) + return; + std::lock_guard lock(thread_group->mutex); if (fatal_error_callback) fatal_error_callback(); diff --git a/src/Common/ThreadStatus.h b/src/Common/ThreadStatus.h index 20550a63312..e620413c8eb 100644 --- a/src/Common/ThreadStatus.h +++ b/src/Common/ThreadStatus.h @@ -290,6 +290,8 @@ public: void logToQueryViewsLog(const ViewRuntimeData & vinfo); + void flushUntrackedMemory(); + protected: void applyQuerySettings(); diff --git a/src/Common/ZooKeeper/ZooKeeper.h b/src/Common/ZooKeeper/ZooKeeper.h index 9de8241cfbe..d20d036f04e 100644 --- a/src/Common/ZooKeeper/ZooKeeper.h +++ b/src/Common/ZooKeeper/ZooKeeper.h @@ -33,6 +33,12 @@ namespace CurrentMetrics namespace DB { class ZooKeeperLog; + +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + } namespace zkutil @@ -79,13 +85,23 @@ concept ZooKeeperResponse = std::derived_from; template struct MultiReadResponses { + MultiReadResponses() = default; + template explicit MultiReadResponses(TResponses responses_) : responses(std::move(responses_)) {} size_t size() const { - return std::visit([](auto && resp) { return resp.size(); }, responses); + return std::visit( + [&](const TResponses & resp) -> size_t + { + if constexpr (std::same_as) + throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "No responses set for MultiRead"); + else + return resp.size(); + }, + responses); } ResponseType & operator[](size_t index) @@ -94,8 +110,10 @@ struct MultiReadResponses [&](TResponses & resp) -> ResponseType & { if constexpr (std::same_as) + { return dynamic_cast(*resp[index]); - else + } + else if constexpr (std::same_as) { if constexpr (try_multi) { @@ -107,6 +125,10 @@ struct MultiReadResponses } return resp[index]; } + else + { + throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "No responses set for MultiRead"); + } }, responses); } @@ -137,7 +159,7 @@ private: size_t size() const { return future_responses.size(); } }; - std::variant responses; + std::variant responses; }; /// ZooKeeper session. The interface is substantially different from the usual libzookeeper API. diff --git a/src/Common/ZooKeeper/ZooKeeperImpl.cpp b/src/Common/ZooKeeper/ZooKeeperImpl.cpp index 1fbdd857379..b637bdea835 100644 --- a/src/Common/ZooKeeper/ZooKeeperImpl.cpp +++ b/src/Common/ZooKeeper/ZooKeeperImpl.cpp @@ -358,12 +358,27 @@ ZooKeeper::ZooKeeper( if (!args.auth_scheme.empty()) sendAuth(args.auth_scheme, args.identity); - send_thread = ThreadFromGlobalPool([this] { sendThread(); }); - receive_thread = ThreadFromGlobalPool([this] { receiveThread(); }); + try + { + send_thread = ThreadFromGlobalPool([this] { sendThread(); }); + receive_thread = ThreadFromGlobalPool([this] { receiveThread(); }); - initApiVersion(); + initApiVersion(); - ProfileEvents::increment(ProfileEvents::ZooKeeperInit); + ProfileEvents::increment(ProfileEvents::ZooKeeperInit); + } + catch (...) + { + tryLogCurrentException(log, "Failed to connect to ZooKeeper"); + + if (send_thread.joinable()) + send_thread.join(); + + if (receive_thread.joinable()) + receive_thread.join(); + + throw; + } } diff --git a/src/Common/ZooKeeper/ZooKeeperLock.cpp b/src/Common/ZooKeeper/ZooKeeperLock.cpp index 1200dcdb533..a52c942a35f 100644 --- a/src/Common/ZooKeeper/ZooKeeperLock.cpp +++ b/src/Common/ZooKeeper/ZooKeeperLock.cpp @@ -41,6 +41,16 @@ ZooKeeperLock::~ZooKeeperLock() } } +bool ZooKeeperLock::isLocked() const +{ + return locked; +} + +const std::string & ZooKeeperLock::getLockPath() const +{ + return lock_path; +} + void ZooKeeperLock::unlock() { if (!locked) diff --git a/src/Common/ZooKeeper/ZooKeeperLock.h b/src/Common/ZooKeeper/ZooKeeperLock.h index f249e69dcc3..755ca1333b8 100644 --- a/src/Common/ZooKeeper/ZooKeeperLock.h +++ b/src/Common/ZooKeeper/ZooKeeperLock.h @@ -37,6 +37,8 @@ public: void unlock(); bool tryLock(); + bool isLocked() const; + const std::string & getLockPath() const; private: zkutil::ZooKeeperPtr zookeeper; diff --git a/src/Common/escapeForFileName.cpp b/src/Common/escapeForFileName.cpp index bcca04706dc..a1f9bff28d0 100644 --- a/src/Common/escapeForFileName.cpp +++ b/src/Common/escapeForFileName.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include diff --git a/src/Common/filesystemHelpers.cpp b/src/Common/filesystemHelpers.cpp index 6e1b5573bef..eabc7bdafbb 100644 --- a/src/Common/filesystemHelpers.cpp +++ b/src/Common/filesystemHelpers.cpp @@ -383,6 +383,14 @@ bool isSymlink(const fs::path & path) return fs::is_symlink(path); /// STYLE_CHECK_ALLOW_STD_FS_SYMLINK } +bool isSymlinkNoThrow(const fs::path & path) +{ + std::error_code dummy; + if (path.filename().empty()) + return fs::is_symlink(path.parent_path(), dummy); /// STYLE_CHECK_ALLOW_STD_FS_SYMLINK + return fs::is_symlink(path, dummy); /// STYLE_CHECK_ALLOW_STD_FS_SYMLINK +} + fs::path readSymlink(const fs::path & path) { /// See the comment for isSymlink diff --git a/src/Common/filesystemHelpers.h b/src/Common/filesystemHelpers.h index 14ee5f54322..8591cd6cf92 100644 --- a/src/Common/filesystemHelpers.h +++ b/src/Common/filesystemHelpers.h @@ -95,6 +95,7 @@ void setModificationTime(const std::string & path, time_t time); time_t getChangeTime(const std::string & path); bool isSymlink(const fs::path & path); +bool isSymlinkNoThrow(const fs::path & path); fs::path readSymlink(const fs::path & path); } diff --git a/src/Common/formatIPv6.cpp b/src/Common/formatIPv6.cpp index 7c027a23b4d..86e33beb7c3 100644 --- a/src/Common/formatIPv6.cpp +++ b/src/Common/formatIPv6.cpp @@ -1,5 +1,5 @@ #include -#include +#include #include #include diff --git a/src/Common/formatIPv6.h b/src/Common/formatIPv6.h index bc8f70f047c..be4dfc7391e 100644 --- a/src/Common/formatIPv6.h +++ b/src/Common/formatIPv6.h @@ -7,7 +7,7 @@ #include #include #include -#include +#include #include constexpr size_t IPV4_BINARY_LENGTH = 4; @@ -82,11 +82,7 @@ inline bool parseIPv4(T * &src, EOFfunction eof, unsigned char * dst, int32_t fi break; } - if constexpr (std::endian::native == std::endian::little) - memcpy(dst, &result, sizeof(result)); - else - reverseMemcpy(dst, &result, sizeof(result)); - + memcpy(dst, &result, sizeof(result)); return true; } diff --git a/src/Common/getHashOfLoadedBinary.cpp b/src/Common/getHashOfLoadedBinary.cpp index da053750036..cc0ad0d2143 100644 --- a/src/Common/getHashOfLoadedBinary.cpp +++ b/src/Common/getHashOfLoadedBinary.cpp @@ -4,7 +4,7 @@ #include #include -#include +#include static int callback(dl_phdr_info * info, size_t, void * data) diff --git a/src/Common/getMappedArea.cpp b/src/Common/getMappedArea.cpp index 573d3194f3d..4f40c604c6a 100644 --- a/src/Common/getMappedArea.cpp +++ b/src/Common/getMappedArea.cpp @@ -4,7 +4,7 @@ #if defined(OS_LINUX) #include -#include +#include #include #include diff --git a/src/Common/hex.cpp b/src/Common/hex.cpp deleted file mode 100644 index e8f9b981062..00000000000 --- a/src/Common/hex.cpp +++ /dev/null @@ -1,92 +0,0 @@ -#include - -const char * const hex_digit_to_char_uppercase_table = "0123456789ABCDEF"; -const char * const hex_digit_to_char_lowercase_table = "0123456789abcdef"; - -const char * const hex_byte_to_char_uppercase_table = - "000102030405060708090A0B0C0D0E0F" - "101112131415161718191A1B1C1D1E1F" - "202122232425262728292A2B2C2D2E2F" - "303132333435363738393A3B3C3D3E3F" - "404142434445464748494A4B4C4D4E4F" - "505152535455565758595A5B5C5D5E5F" - "606162636465666768696A6B6C6D6E6F" - "707172737475767778797A7B7C7D7E7F" - "808182838485868788898A8B8C8D8E8F" - "909192939495969798999A9B9C9D9E9F" - "A0A1A2A3A4A5A6A7A8A9AAABACADAEAF" - "B0B1B2B3B4B5B6B7B8B9BABBBCBDBEBF" - "C0C1C2C3C4C5C6C7C8C9CACBCCCDCECF" - "D0D1D2D3D4D5D6D7D8D9DADBDCDDDEDF" - "E0E1E2E3E4E5E6E7E8E9EAEBECEDEEEF" - "F0F1F2F3F4F5F6F7F8F9FAFBFCFDFEFF"; - -const char * const hex_byte_to_char_lowercase_table = - "000102030405060708090a0b0c0d0e0f" - "101112131415161718191a1b1c1d1e1f" - "202122232425262728292a2b2c2d2e2f" - "303132333435363738393a3b3c3d3e3f" - "404142434445464748494a4b4c4d4e4f" - "505152535455565758595a5b5c5d5e5f" - "606162636465666768696a6b6c6d6e6f" - "707172737475767778797a7b7c7d7e7f" - "808182838485868788898a8b8c8d8e8f" - "909192939495969798999a9b9c9d9e9f" - "a0a1a2a3a4a5a6a7a8a9aaabacadaeaf" - "b0b1b2b3b4b5b6b7b8b9babbbcbdbebf" - "c0c1c2c3c4c5c6c7c8c9cacbcccdcecf" - "d0d1d2d3d4d5d6d7d8d9dadbdcdddedf" - "e0e1e2e3e4e5e6e7e8e9eaebecedeeef" - "f0f1f2f3f4f5f6f7f8f9fafbfcfdfeff"; - -const char * const hex_char_to_digit_table = - "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" - "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\xff\xff\xff\xff\xff\xff" //0-9 - "\xff\x0a\x0b\x0c\x0d\x0e\x0f\xff\xff\xff\xff\xff\xff\xff\xff\xff" //A-Z - "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\x0a\x0b\x0c\x0d\x0e\x0f\xff\xff\xff\xff\xff\xff\xff\xff\xff" //a-z - "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"; - -const char * const bin_byte_to_char_table = - "0000000000000001000000100000001100000100000001010000011000000111" - "0000100000001001000010100000101100001100000011010000111000001111" - "0001000000010001000100100001001100010100000101010001011000010111" - "0001100000011001000110100001101100011100000111010001111000011111" - "0010000000100001001000100010001100100100001001010010011000100111" - "0010100000101001001010100010101100101100001011010010111000101111" - "0011000000110001001100100011001100110100001101010011011000110111" - "0011100000111001001110100011101100111100001111010011111000111111" - "0100000001000001010000100100001101000100010001010100011001000111" - "0100100001001001010010100100101101001100010011010100111001001111" - "0101000001010001010100100101001101010100010101010101011001010111" - "0101100001011001010110100101101101011100010111010101111001011111" - "0110000001100001011000100110001101100100011001010110011001100111" - "0110100001101001011010100110101101101100011011010110111001101111" - "0111000001110001011100100111001101110100011101010111011001110111" - "0111100001111001011110100111101101111100011111010111111001111111" - "1000000010000001100000101000001110000100100001011000011010000111" - "1000100010001001100010101000101110001100100011011000111010001111" - "1001000010010001100100101001001110010100100101011001011010010111" - "1001100010011001100110101001101110011100100111011001111010011111" - "1010000010100001101000101010001110100100101001011010011010100111" - "1010100010101001101010101010101110101100101011011010111010101111" - "1011000010110001101100101011001110110100101101011011011010110111" - "1011100010111001101110101011101110111100101111011011111010111111" - "1100000011000001110000101100001111000100110001011100011011000111" - "1100100011001001110010101100101111001100110011011100111011001111" - "1101000011010001110100101101001111010100110101011101011011010111" - "1101100011011001110110101101101111011100110111011101111011011111" - "1110000011100001111000101110001111100100111001011110011011100111" - "1110100011101001111010101110101111101100111011011110111011101111" - "1111000011110001111100101111001111110100111101011111011011110111" - "1111100011111001111110101111101111111100111111011111111011111111"; diff --git a/src/Common/hex.h b/src/Common/hex.h deleted file mode 100644 index 062a6c27f76..00000000000 --- a/src/Common/hex.h +++ /dev/null @@ -1,145 +0,0 @@ -#pragma once -#include - - -/// Maps 0..15 to 0..9A..F or 0..9a..f correspondingly. - -extern const char * const hex_digit_to_char_uppercase_table; -extern const char * const hex_digit_to_char_lowercase_table; - -inline char hexDigitUppercase(unsigned char c) -{ - return hex_digit_to_char_uppercase_table[c]; -} - -inline char hexDigitLowercase(unsigned char c) -{ - return hex_digit_to_char_lowercase_table[c]; -} - - -#include -#include - -#include - - -/// Maps 0..255 to 00..FF or 00..ff correspondingly - -extern const char * const hex_byte_to_char_uppercase_table; -extern const char * const hex_byte_to_char_lowercase_table; - -inline void writeHexByteUppercase(UInt8 byte, void * out) -{ - memcpy(out, &hex_byte_to_char_uppercase_table[static_cast(byte) * 2], 2); -} - -inline void writeHexByteLowercase(UInt8 byte, void * out) -{ - memcpy(out, &hex_byte_to_char_lowercase_table[static_cast(byte) * 2], 2); -} - -extern const char * const bin_byte_to_char_table; - -inline void writeBinByte(UInt8 byte, void * out) -{ - memcpy(out, &bin_byte_to_char_table[static_cast(byte) * 8], 8); -} - -/// Produces hex representation of an unsigned int with leading zeros (for checksums) -template -inline void writeHexUIntImpl(TUInt uint_, char * out, const char * const table) -{ - union - { - TUInt value; - UInt8 uint8[sizeof(TUInt)]; - }; - - value = uint_; - - for (size_t i = 0; i < sizeof(TUInt); ++i) - { - if constexpr (std::endian::native == std::endian::little) - memcpy(out + i * 2, &table[static_cast(uint8[sizeof(TUInt) - 1 - i]) * 2], 2); - else - memcpy(out + i * 2, &table[static_cast(uint8[i]) * 2], 2); - } -} - -template -inline void writeHexUIntUppercase(TUInt uint_, char * out) -{ - writeHexUIntImpl(uint_, out, hex_byte_to_char_uppercase_table); -} - -template -inline void writeHexUIntLowercase(TUInt uint_, char * out) -{ - writeHexUIntImpl(uint_, out, hex_byte_to_char_lowercase_table); -} - -template -std::string getHexUIntUppercase(TUInt uint_) -{ - std::string res(sizeof(TUInt) * 2, '\0'); - writeHexUIntUppercase(uint_, res.data()); - return res; -} - -template -std::string getHexUIntLowercase(TUInt uint_) -{ - std::string res(sizeof(TUInt) * 2, '\0'); - writeHexUIntLowercase(uint_, res.data()); - return res; -} - - -/// Maps 0..9, A..F, a..f to 0..15. Other chars are mapped to implementation specific value. - -extern const char * const hex_char_to_digit_table; - -inline UInt8 unhex(char c) -{ - return hex_char_to_digit_table[static_cast(c)]; -} - -inline UInt8 unhex2(const char * data) -{ - return - static_cast(unhex(data[0])) * 0x10 - + static_cast(unhex(data[1])); -} - -inline UInt16 unhex4(const char * data) -{ - return - static_cast(unhex(data[0])) * 0x1000 - + static_cast(unhex(data[1])) * 0x100 - + static_cast(unhex(data[2])) * 0x10 - + static_cast(unhex(data[3])); -} - -template -TUInt unhexUInt(const char * data) -{ - TUInt res = 0; - if constexpr ((sizeof(TUInt) <= 8) || ((sizeof(TUInt) % 8) != 0)) - { - for (size_t i = 0; i < sizeof(TUInt) * 2; ++i, ++data) - { - res <<= 4; - res += unhex(*data); - } - } - else - { - for (size_t i = 0; i < sizeof(TUInt) / 8; ++i, data += 16) - { - res <<= 64; - res += unhexUInt(data); - } - } - return res; -} diff --git a/src/Common/interpolate.h b/src/Common/interpolate.h deleted file mode 100644 index 05900563b80..00000000000 --- a/src/Common/interpolate.h +++ /dev/null @@ -1,20 +0,0 @@ -#pragma once - -#include - - -inline double interpolateLinear(double min, double max, double ratio) -{ - return min + (max - min) * ratio; -} - - -/** It is linear interpolation in logarithmic coordinates. - * Exponential interpolation is related to linear interpolation - * exactly in same way as geometric mean is related to arithmetic mean. - * 'min' must be greater than zero, 'ratio' must be from 0 to 1. - */ -inline double interpolateExponential(double min, double max, double ratio) -{ - return min * std::pow(max / min, ratio); -} diff --git a/src/Common/logger_useful.h b/src/Common/logger_useful.h index 170ca1c3b10..2a0c582331d 100644 --- a/src/Common/logger_useful.h +++ b/src/Common/logger_useful.h @@ -10,35 +10,16 @@ namespace Poco { class Logger; } -/// This wrapper is useful to save formatted message into a String before sending it to a logger -class LogToStrImpl -{ - String & out_str; - Poco::Logger * logger; - bool propagate_to_actual_log = true; -public: - LogToStrImpl(String & out_str_, Poco::Logger * logger_) : out_str(out_str_) , logger(logger_) {} - LogToStrImpl & operator -> () { return *this; } - bool is(Poco::Message::Priority priority) { propagate_to_actual_log &= logger->is(priority); return true; } - LogToStrImpl * getChannel() {return this; } - const String & name() const { return logger->name(); } - void log(const Poco::Message & message) - { - out_str = message.getText(); - if (!propagate_to_actual_log) - return; - if (auto * channel = logger->getChannel()) - channel->log(message); - } -}; #define LogToStr(x, y) std::make_unique(x, y) +#define LogFrequencyLimiter(x, y) std::make_unique(x, y) namespace { [[maybe_unused]] const ::Poco::Logger * getLogger(const ::Poco::Logger * logger) { return logger; }; [[maybe_unused]] const ::Poco::Logger * getLogger(const std::atomic<::Poco::Logger *> & logger) { return logger.load(); }; [[maybe_unused]] std::unique_ptr getLogger(std::unique_ptr && logger) { return logger; }; + [[maybe_unused]] std::unique_ptr getLogger(std::unique_ptr && logger) { return logger; }; } #define LOG_IMPL_FIRST_ARG(X, ...) X diff --git a/src/Common/tests/gtest_find_symbols.cpp b/src/Common/tests/gtest_find_symbols.cpp index 1daab982d01..3c19549eb75 100644 --- a/src/Common/tests/gtest_find_symbols.cpp +++ b/src/Common/tests/gtest_find_symbols.cpp @@ -4,6 +4,15 @@ #include +template +void test_find_first_not(const std::string & haystack, std::size_t expected_pos) +{ + const char * begin = haystack.data(); + const char * end = haystack.data() + haystack.size(); + + ASSERT_EQ(begin + expected_pos, find_first_not_symbols(begin, end)); +} + TEST(FindSymbols, SimpleTest) { std::string s = "Hello, world! Goodbye..."; @@ -36,3 +45,58 @@ TEST(FindSymbols, SimpleTest) ASSERT_EQ(vals, (std::vector{"s", "String"})); } } + +TEST(FindNotSymbols, AllSymbolsPresent) +{ + std::string str_with_17_bytes = "hello world hello"; + std::string str_with_16_bytes = {str_with_17_bytes.begin(), str_with_17_bytes.end() - 1u}; + std::string str_with_15_bytes = {str_with_16_bytes.begin(), str_with_16_bytes.end() - 1u}; + + /* + * The below variations will choose different implementation strategies: + * 1. Loop method only because it does not contain enough bytes for SSE 4.2 + * 2. SSE4.2 only since string contains exactly 16 bytes + * 3. SSE4.2 + Loop method will take place because only first 16 bytes are treated by SSE 4.2 and remaining bytes is treated by loop + * + * Below code asserts that all calls return the ::end of the input string. This was not true prior to this fix as mentioned in PR #47304 + * */ + + test_find_first_not<'h', 'e', 'l', 'o', 'w', 'r', 'd', ' '>(str_with_15_bytes, str_with_15_bytes.size()); + test_find_first_not<'h', 'e', 'l', 'o', 'w', 'r', 'd', ' '>(str_with_16_bytes, str_with_16_bytes.size()); + test_find_first_not<'h', 'e', 'l', 'o', 'w', 'r', 'd', ' '>(str_with_17_bytes, str_with_17_bytes.size()); +} + +TEST(FindNotSymbols, NoSymbolsMatch) +{ + std::string s = "abcdefg"; + + // begin should be returned since the first character of the string does not match any of the below symbols + test_find_first_not<'h', 'i', 'j'>(s, 0u); +} + +TEST(FindNotSymbols, ExtraSymbols) +{ + std::string s = "hello_world_hello"; + test_find_first_not<'h', 'e', 'l', 'o', ' '>(s, 5u); +} + +TEST(FindNotSymbols, EmptyString) +{ + std::string s; + test_find_first_not<'h', 'e', 'l', 'o', 'w', 'r', 'd', ' '>(s, s.size()); +} + +TEST(FindNotSymbols, SingleChar) +{ + std::string s = "a"; + test_find_first_not<'a'>(s, s.size()); +} + +TEST(FindNotSymbols, NullCharacter) +{ + // special test to ensure only the passed template arguments are used as needles + // since current find_first_symbols implementation takes in 16 characters and defaults + // to \0. + std::string s("abcdefg\0x", 9u); + test_find_first_not<'a', 'b', 'c', 'd', 'e', 'f', 'g'>(s, 7u); +} diff --git a/src/Compression/CompressedReadBufferBase.cpp b/src/Compression/CompressedReadBufferBase.cpp index ab856cc9801..3111f649b26 100644 --- a/src/Compression/CompressedReadBufferBase.cpp +++ b/src/Compression/CompressedReadBufferBase.cpp @@ -6,7 +6,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/src/Compression/CompressionCodecDelta.cpp b/src/Compression/CompressionCodecDelta.cpp index 6d6078b9ee1..37f9230da14 100644 --- a/src/Compression/CompressionCodecDelta.cpp +++ b/src/Compression/CompressionCodecDelta.cpp @@ -193,7 +193,8 @@ void registerCodecDelta(CompressionCodecFactory & factory) UInt8 method_code = static_cast(CompressionMethodByte::Delta); auto codec_builder = [&](const ASTPtr & arguments, const IDataType * column_type) -> CompressionCodecPtr { - UInt8 delta_bytes_size = 0; + /// Default bytes size is 1. + UInt8 delta_bytes_size = 1; if (arguments && !arguments->children.empty()) { @@ -202,8 +203,8 @@ void registerCodecDelta(CompressionCodecFactory & factory) const auto children = arguments->children; const auto * literal = children[0]->as(); - if (!literal) - throw Exception(ErrorCodes::ILLEGAL_CODEC_PARAMETER, "Delta codec argument must be integer"); + if (!literal || literal->value.getType() != Field::Types::Which::UInt64) + throw Exception(ErrorCodes::ILLEGAL_CODEC_PARAMETER, "Delta codec argument must be unsigned integer"); size_t user_bytes_size = literal->value.safeGet(); if (user_bytes_size != 1 && user_bytes_size != 2 && user_bytes_size != 4 && user_bytes_size != 8) diff --git a/src/Compression/CompressionCodecDoubleDelta.cpp b/src/Compression/CompressionCodecDoubleDelta.cpp index 782675dfd32..dea15f99a5a 100644 --- a/src/Compression/CompressionCodecDoubleDelta.cpp +++ b/src/Compression/CompressionCodecDoubleDelta.cpp @@ -7,7 +7,7 @@ #include #include #include -#include +#include #include #include @@ -31,7 +31,7 @@ namespace DB /** DoubleDelta column codec implementation. * * Based on Gorilla paper: http://www.vldb.org/pvldb/vol8/p1816-teller.pdf, which was extended - * to support 64bit types. The drawback is 1 extra bit for 32-byte wide deltas: 5-bit prefix + * to support 64bit types. The drawback is 1 extra bit for 32-bit wide deltas: 5-bit prefix * instead of 4-bit prefix. * * This codec is best used against monotonic integer sequences with constant (or almost constant) @@ -145,6 +145,8 @@ namespace ErrorCodes extern const int CANNOT_COMPRESS; extern const int CANNOT_DECOMPRESS; extern const int BAD_ARGUMENTS; + extern const int ILLEGAL_SYNTAX_FOR_CODEC_TYPE; + extern const int ILLEGAL_CODEC_PARAMETER; } namespace @@ -549,10 +551,28 @@ void registerCodecDoubleDelta(CompressionCodecFactory & factory) factory.registerCompressionCodecWithType("DoubleDelta", method_code, [&](const ASTPtr & arguments, const IDataType * column_type) -> CompressionCodecPtr { - if (arguments) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Codec DoubleDelta does not accept any arguments"); + /// Default bytes size is 1. + UInt8 data_bytes_size = 1; + if (arguments && !arguments->children.empty()) + { + if (arguments->children.size() > 1) + throw Exception(ErrorCodes::ILLEGAL_SYNTAX_FOR_CODEC_TYPE, "DoubleDelta codec must have 1 parameter, given {}", arguments->children.size()); + + const auto children = arguments->children; + const auto * literal = children[0]->as(); + if (!literal || literal->value.getType() != Field::Types::Which::UInt64) + throw Exception(ErrorCodes::ILLEGAL_CODEC_PARAMETER, "DoubleDelta codec argument must be unsigned integer"); + + size_t user_bytes_size = literal->value.safeGet(); + if (user_bytes_size != 1 && user_bytes_size != 2 && user_bytes_size != 4 && user_bytes_size != 8) + throw Exception(ErrorCodes::ILLEGAL_CODEC_PARAMETER, "Argument value for DoubleDelta codec can be 1, 2, 4 or 8, given {}", user_bytes_size); + data_bytes_size = static_cast(user_bytes_size); + } + else if (column_type) + { + data_bytes_size = getDataBytesSize(column_type); + } - UInt8 data_bytes_size = column_type ? getDataBytesSize(column_type) : 0; return std::make_shared(data_bytes_size); }); } diff --git a/src/Compression/CompressionCodecFPC.cpp b/src/Compression/CompressionCodecFPC.cpp index 31b12b762c8..8c3e518ed62 100644 --- a/src/Compression/CompressionCodecFPC.cpp +++ b/src/Compression/CompressionCodecFPC.cpp @@ -109,28 +109,42 @@ void registerCodecFPC(CompressionCodecFactory & factory) auto method_code = static_cast(CompressionMethodByte::FPC); auto codec_builder = [&](const ASTPtr & arguments, const IDataType * column_type) -> CompressionCodecPtr { - UInt8 float_width = 0; + /// Set default float width to 4. + UInt8 float_width = 4; if (column_type != nullptr) float_width = getFloatBytesSize(*column_type); UInt8 level = CompressionCodecFPC::DEFAULT_COMPRESSION_LEVEL; if (arguments && !arguments->children.empty()) { - if (arguments->children.size() > 1) + if (arguments->children.size() > 2) { throw Exception(ErrorCodes::ILLEGAL_SYNTAX_FOR_CODEC_TYPE, - "FPC codec must have 1 parameter, given {}", arguments->children.size()); + "FPC codec must have from 0 to 2 parameters, given {}", arguments->children.size()); } const auto * literal = arguments->children.front()->as(); - if (!literal) - throw Exception(ErrorCodes::ILLEGAL_CODEC_PARAMETER, "FPC codec argument must be integer"); + if (!literal || literal->value.getType() != Field::Types::Which::UInt64) + throw Exception(ErrorCodes::ILLEGAL_CODEC_PARAMETER, "FPC codec argument must be unsigned integer"); level = literal->value.safeGet(); if (level < 1 || level > CompressionCodecFPC::MAX_COMPRESSION_LEVEL) throw Exception(ErrorCodes::ILLEGAL_CODEC_PARAMETER, "FPC codec level must be between {} and {}", 1, static_cast(CompressionCodecFPC::MAX_COMPRESSION_LEVEL)); + + if (arguments->children.size() == 2) + { + literal = arguments->children[1]->as(); + if (!literal || !isInt64OrUInt64FieldType(literal->value.getType())) + throw Exception(ErrorCodes::ILLEGAL_CODEC_PARAMETER, "FPC codec argument must be unsigned integer"); + + size_t user_float_width = literal->value.safeGet(); + if (user_float_width != 4 && user_float_width != 8) + throw Exception(ErrorCodes::ILLEGAL_CODEC_PARAMETER, "Float size for FPC codec can be 4 or 8, given {}", user_float_width); + float_width = static_cast(user_float_width); + } } + return std::make_shared(float_width, level); }; factory.registerCompressionCodecWithType("FPC", method_code, codec_builder); diff --git a/src/Compression/CompressionCodecGorilla.cpp b/src/Compression/CompressionCodecGorilla.cpp index d68648bd83c..568640153ac 100644 --- a/src/Compression/CompressionCodecGorilla.cpp +++ b/src/Compression/CompressionCodecGorilla.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -134,6 +135,8 @@ namespace ErrorCodes extern const int CANNOT_COMPRESS; extern const int CANNOT_DECOMPRESS; extern const int BAD_ARGUMENTS; + extern const int ILLEGAL_SYNTAX_FOR_CODEC_TYPE; + extern const int ILLEGAL_CODEC_PARAMETER; } namespace @@ -445,10 +448,28 @@ void registerCodecGorilla(CompressionCodecFactory & factory) UInt8 method_code = static_cast(CompressionMethodByte::Gorilla); auto codec_builder = [&](const ASTPtr & arguments, const IDataType * column_type) -> CompressionCodecPtr { - if (arguments) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Codec Gorilla does not accept any arguments"); + /// Default bytes size is 1 + UInt8 data_bytes_size = 1; + if (arguments && !arguments->children.empty()) + { + if (arguments->children.size() > 1) + throw Exception(ErrorCodes::ILLEGAL_SYNTAX_FOR_CODEC_TYPE, "Gorilla codec must have 1 parameter, given {}", arguments->children.size()); + + const auto children = arguments->children; + const auto * literal = children[0]->as(); + if (!literal || literal->value.getType() != Field::Types::Which::UInt64) + throw Exception(ErrorCodes::ILLEGAL_CODEC_PARAMETER, "Gorilla codec argument must be unsigned integer"); + + size_t user_bytes_size = literal->value.safeGet(); + if (user_bytes_size != 1 && user_bytes_size != 2 && user_bytes_size != 4 && user_bytes_size != 8) + throw Exception(ErrorCodes::ILLEGAL_CODEC_PARAMETER, "Argument value for Gorilla codec can be 1, 2, 4 or 8, given {}", user_bytes_size); + data_bytes_size = static_cast(user_bytes_size); + } + else if (column_type) + { + data_bytes_size = getDataBytesSize(column_type); + } - UInt8 data_bytes_size = column_type ? getDataBytesSize(column_type) : 0; return std::make_shared(data_bytes_size); }; factory.registerCompressionCodecWithType("Gorilla", method_code, codec_builder); diff --git a/src/Compression/CompressionCodecMultiple.cpp b/src/Compression/CompressionCodecMultiple.cpp index 5203e349317..dba67749e4d 100644 --- a/src/Compression/CompressionCodecMultiple.cpp +++ b/src/Compression/CompressionCodecMultiple.cpp @@ -8,7 +8,7 @@ #include #include #include -#include +#include namespace DB diff --git a/src/Compression/CompressionCodecT64.cpp b/src/Compression/CompressionCodecT64.cpp index e7f1615128a..1f8331c8a5f 100644 --- a/src/Compression/CompressionCodecT64.cpp +++ b/src/Compression/CompressionCodecT64.cpp @@ -33,7 +33,8 @@ public: Bit }; - CompressionCodecT64(TypeIndex type_idx_, Variant variant_); + // type_idx_ is required for compression, but not for decompression. + CompressionCodecT64(std::optional type_idx_, Variant variant_); uint8_t getMethodByte() const override; @@ -53,7 +54,7 @@ protected: bool isGenericCompression() const override { return false; } private: - TypeIndex type_idx; + std::optional type_idx; Variant variant; }; @@ -91,9 +92,12 @@ enum class MagicNumber : uint8_t IPv4 = 21, }; -MagicNumber serializeTypeId(TypeIndex type_id) +MagicNumber serializeTypeId(std::optional type_id) { - switch (type_id) + if (!type_id) + throw Exception(ErrorCodes::CANNOT_COMPRESS, "T64 codec doesn't support compression without information about column type"); + + switch (*type_id) { case TypeIndex::UInt8: return MagicNumber::UInt8; case TypeIndex::UInt16: return MagicNumber::UInt16; @@ -115,7 +119,7 @@ MagicNumber serializeTypeId(TypeIndex type_id) break; } - throw Exception(ErrorCodes::LOGICAL_ERROR, "Type is not supported by T64 codec: {}", static_cast(type_id)); + throw Exception(ErrorCodes::LOGICAL_ERROR, "Type is not supported by T64 codec: {}", static_cast(*type_id)); } TypeIndex deserializeTypeId(uint8_t serialized_type_id) @@ -632,7 +636,7 @@ UInt32 CompressionCodecT64::doCompressData(const char * src, UInt32 src_size, ch memcpy(dst, &cookie, 1); dst += 1; - switch (baseType(type_idx)) + switch (baseType(*type_idx)) { case TypeIndex::Int8: return 1 + compressData(src, src_size, dst, variant); @@ -699,7 +703,7 @@ uint8_t CompressionCodecT64::getMethodByte() const return codecId(); } -CompressionCodecT64::CompressionCodecT64(TypeIndex type_idx_, Variant variant_) +CompressionCodecT64::CompressionCodecT64(std::optional type_idx_, Variant variant_) : type_idx(type_idx_) , variant(variant_) { @@ -712,7 +716,7 @@ CompressionCodecT64::CompressionCodecT64(TypeIndex type_idx_, Variant variant_) void CompressionCodecT64::updateHash(SipHash & hash) const { getCodecDesc()->updateTreeHash(hash); - hash.update(type_idx); + hash.update(type_idx.value_or(TypeIndex::Nothing)); hash.update(variant); } @@ -742,9 +746,14 @@ void registerCodecT64(CompressionCodecFactory & factory) throw Exception(ErrorCodes::ILLEGAL_CODEC_PARAMETER, "Wrong modification for T64: {}", name); } - auto type_idx = typeIdx(type); - if (type && type_idx == TypeIndex::Nothing) - throw Exception(ErrorCodes::ILLEGAL_SYNTAX_FOR_CODEC_TYPE, "T64 codec is not supported for specified type {}", type->getName()); + std::optional type_idx; + if (type) + { + type_idx = typeIdx(type); + if (type_idx == TypeIndex::Nothing) + throw Exception( + ErrorCodes::ILLEGAL_SYNTAX_FOR_CODEC_TYPE, "T64 codec is not supported for specified type {}", type->getName()); + } return std::make_shared(type_idx, variant); }; diff --git a/src/Coordination/Changelog.cpp b/src/Coordination/Changelog.cpp index 899310dc591..ddedae4fa0f 100644 --- a/src/Coordination/Changelog.cpp +++ b/src/Coordination/Changelog.cpp @@ -211,9 +211,14 @@ public: void flush() { auto * file_buffer = tryGetFileBuffer(); - /// Fsync file system if needed - if (file_buffer && log_file_settings.force_sync) - file_buffer->sync(); + if (file_buffer) + { + /// Fsync file system if needed + if (log_file_settings.force_sync) + file_buffer->sync(); + else + file_buffer->next(); + } } uint64_t getStartIndex() const diff --git a/src/Coordination/CoordinationSettings.cpp b/src/Coordination/CoordinationSettings.cpp index e665ccb89c7..5e1ac1e2d7f 100644 --- a/src/Coordination/CoordinationSettings.cpp +++ b/src/Coordination/CoordinationSettings.cpp @@ -36,7 +36,7 @@ void CoordinationSettings::loadFromConfig(const String & config_elem, const Poco } -const String KeeperConfigurationAndSettings::DEFAULT_FOUR_LETTER_WORD_CMD = "conf,cons,crst,envi,ruok,srst,srvr,stat,wchs,dirs,mntr,isro,rcvr,apiv,csnp,lgif,rqld"; +const String KeeperConfigurationAndSettings::DEFAULT_FOUR_LETTER_WORD_CMD = "conf,cons,crst,envi,ruok,srst,srvr,stat,wchs,dirs,mntr,isro,rcvr,apiv,csnp,lgif,rqld,rclc,clrs"; KeeperConfigurationAndSettings::KeeperConfigurationAndSettings() : server_id(NOT_EXIST) diff --git a/src/Coordination/FourLetterCommand.cpp b/src/Coordination/FourLetterCommand.cpp index 3e0e5acee0c..8a7fdb82fb7 100644 --- a/src/Coordination/FourLetterCommand.cpp +++ b/src/Coordination/FourLetterCommand.cpp @@ -145,6 +145,12 @@ void FourLetterCommandFactory::registerCommands(KeeperDispatcher & keeper_dispat FourLetterCommandPtr request_leader_command = std::make_shared(keeper_dispatcher); factory.registerCommand(request_leader_command); + FourLetterCommandPtr recalculate_command = std::make_shared(keeper_dispatcher); + factory.registerCommand(recalculate_command); + + FourLetterCommandPtr clean_resources_command = std::make_shared(keeper_dispatcher); + factory.registerCommand(clean_resources_command); + factory.initializeAllowList(keeper_dispatcher); factory.setInitialize(true); } @@ -515,4 +521,16 @@ String RequestLeaderCommand::run() return keeper_dispatcher.requestLeader() ? "Sent leadership request to leader." : "Failed to send leadership request to leader."; } +String RecalculateCommand::run() +{ + keeper_dispatcher.recalculateStorageStats(); + return "ok"; +} + +String CleanResourcesCommand::run() +{ + keeper_dispatcher.cleanResources(); + return "ok"; +} + } diff --git a/src/Coordination/FourLetterCommand.h b/src/Coordination/FourLetterCommand.h index 8a8aacf7a3a..c1a91303c05 100644 --- a/src/Coordination/FourLetterCommand.h +++ b/src/Coordination/FourLetterCommand.h @@ -377,4 +377,28 @@ struct RequestLeaderCommand : public IFourLetterCommand ~RequestLeaderCommand() override = default; }; +struct RecalculateCommand : public IFourLetterCommand +{ + explicit RecalculateCommand(KeeperDispatcher & keeper_dispatcher_) + : IFourLetterCommand(keeper_dispatcher_) + { + } + + String name() override { return "rclc"; } + String run() override; + ~RecalculateCommand() override = default; +}; + +struct CleanResourcesCommand : public IFourLetterCommand +{ + explicit CleanResourcesCommand(KeeperDispatcher & keeper_dispatcher_) + : IFourLetterCommand(keeper_dispatcher_) + { + } + + String name() override { return "clrs"; } + String run() override; + ~CleanResourcesCommand() override = default; +}; + } diff --git a/src/Coordination/KeeperDispatcher.cpp b/src/Coordination/KeeperDispatcher.cpp index 06c693e45be..2aa11dd9eed 100644 --- a/src/Coordination/KeeperDispatcher.cpp +++ b/src/Coordination/KeeperDispatcher.cpp @@ -4,12 +4,12 @@ #include #include -#include +#include #include #include #include #include - +#include #include #include @@ -17,12 +17,26 @@ #include #include +#if USE_JEMALLOC +# include + +#define STRINGIFY_HELPER(x) #x +#define STRINGIFY(x) STRINGIFY_HELPER(x) + +#endif + namespace CurrentMetrics { extern const Metric KeeperAliveConnections; extern const Metric KeeperOutstandingRequets; } +namespace ProfileEvents +{ + extern const Event MemoryAllocatorPurge; + extern const Event MemoryAllocatorPurgeTimeMicroseconds; +} + namespace fs = std::filesystem; namespace DB @@ -753,4 +767,15 @@ Keeper4LWInfo KeeperDispatcher::getKeeper4LWInfo() const return result; } +void KeeperDispatcher::cleanResources() +{ +#if USE_JEMALLOC + LOG_TRACE(&Poco::Logger::get("KeeperDispatcher"), "Purging unused memory"); + Stopwatch watch; + mallctl("arena." STRINGIFY(MALLCTL_ARENAS_ALL) ".purge", nullptr, nullptr, nullptr, 0); + ProfileEvents::increment(ProfileEvents::MemoryAllocatorPurge); + ProfileEvents::increment(ProfileEvents::MemoryAllocatorPurgeTimeMicroseconds, watch.elapsedMicroseconds()); +#endif +} + } diff --git a/src/Coordination/KeeperDispatcher.h b/src/Coordination/KeeperDispatcher.h index ff902d8e036..9371d2fbbac 100644 --- a/src/Coordination/KeeperDispatcher.h +++ b/src/Coordination/KeeperDispatcher.h @@ -225,6 +225,13 @@ public: { return server->requestLeader(); } + + void recalculateStorageStats() + { + return server->recalculateStorageStats(); + } + + static void cleanResources(); }; } diff --git a/src/Coordination/KeeperServer.cpp b/src/Coordination/KeeperServer.cpp index 0f131750109..78a095f8c8d 100644 --- a/src/Coordination/KeeperServer.cpp +++ b/src/Coordination/KeeperServer.cpp @@ -960,4 +960,9 @@ bool KeeperServer::requestLeader() return isLeader() || raft_instance->request_leadership(); } +void KeeperServer::recalculateStorageStats() +{ + state_machine->recalculateStorageStats(); +} + } diff --git a/src/Coordination/KeeperServer.h b/src/Coordination/KeeperServer.h index feadf3bb7ce..bcff81f66f2 100644 --- a/src/Coordination/KeeperServer.h +++ b/src/Coordination/KeeperServer.h @@ -137,6 +137,8 @@ public: KeeperLogInfo getKeeperLogInfo(); bool requestLeader(); + + void recalculateStorageStats(); }; } diff --git a/src/Coordination/KeeperSnapshotManager.cpp b/src/Coordination/KeeperSnapshotManager.cpp index a3f7dbc2c6a..8b80db3e520 100644 --- a/src/Coordination/KeeperSnapshotManager.cpp +++ b/src/Coordination/KeeperSnapshotManager.cpp @@ -361,19 +361,25 @@ void KeeperStorageSnapshot::deserialize(SnapshotDeserializationResult & deserial "If you still want to ignore it, you can set 'keeper_server.ignore_system_path_on_startup' to true", error_msg); } - else if (match_result == EXACT && !is_node_empty(node)) + else if (match_result == EXACT) { - if (keeper_context->ignore_system_path_on_startup || keeper_context->server_state != KeeperContext::Phase::INIT) + if (!is_node_empty(node)) { - LOG_ERROR(&Poco::Logger::get("KeeperSnapshotManager"), "{}. Ignoring it", error_msg); - node = KeeperStorage::Node{}; + if (keeper_context->ignore_system_path_on_startup || keeper_context->server_state != KeeperContext::Phase::INIT) + { + LOG_ERROR(&Poco::Logger::get("KeeperSnapshotManager"), "{}. Ignoring it", error_msg); + node = KeeperStorage::Node{}; + } + else + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "{}. Ignoring it can lead to data loss. " + "If you still want to ignore it, you can set 'keeper_server.ignore_system_path_on_startup' to true", + error_msg); } - else - throw Exception( - ErrorCodes::LOGICAL_ERROR, - "{}. Ignoring it can lead to data loss. " - "If you still want to ignore it, you can set 'keeper_server.ignore_system_path_on_startup' to true", - error_msg); + + // we always ignore the written size for this node + node.recalculateSize(); } storage.container.insertOrReplace(path, node); @@ -390,7 +396,7 @@ void KeeperStorageSnapshot::deserialize(SnapshotDeserializationResult & deserial { auto parent_path = parentPath(itr.key); storage.container.updateValue( - parent_path, [path = itr.key](KeeperStorage::Node & value) { value.addChild(getBaseName(path)); }); + parent_path, [version, path = itr.key](KeeperStorage::Node & value) { value.addChild(getBaseName(path), /*update_size*/ version < SnapshotVersion::V4); }); } } diff --git a/src/Coordination/KeeperStateMachine.cpp b/src/Coordination/KeeperStateMachine.cpp index ccee058ba5c..d0bd18b63e2 100644 --- a/src/Coordination/KeeperStateMachine.cpp +++ b/src/Coordination/KeeperStateMachine.cpp @@ -643,4 +643,12 @@ ClusterConfigPtr KeeperStateMachine::getClusterConfig() const return nullptr; } +void KeeperStateMachine::recalculateStorageStats() +{ + std::lock_guard lock(storage_and_responses_lock); + LOG_INFO(log, "Recalculating storage stats"); + storage->recalculateStats(); + LOG_INFO(log, "Done recalculating storage stats"); +} + } diff --git a/src/Coordination/KeeperStateMachine.h b/src/Coordination/KeeperStateMachine.h index ffc7fce1cfe..d8181532f09 100644 --- a/src/Coordination/KeeperStateMachine.h +++ b/src/Coordination/KeeperStateMachine.h @@ -103,6 +103,7 @@ public: uint64_t getKeyArenaSize() const; uint64_t getLatestSnapshotBufSize() const; + void recalculateStorageStats(); private: /// In our state machine we always have a single snapshot which is stored /// in memory in compressed (serialized) format. diff --git a/src/Coordination/KeeperStorage.cpp b/src/Coordination/KeeperStorage.cpp index 0e4d631938c..41a6af54204 100644 --- a/src/Coordination/KeeperStorage.cpp +++ b/src/Coordination/KeeperStorage.cpp @@ -11,7 +11,7 @@ #include #include #include -#include +#include #include #include #include @@ -201,9 +201,10 @@ void KeeperStorage::Node::setData(String new_data) data = std::move(new_data); } -void KeeperStorage::Node::addChild(StringRef child_path) +void KeeperStorage::Node::addChild(StringRef child_path, bool update_size) { - size_bytes += sizeof child_path; + if (update_size) [[likely]] + size_bytes += sizeof child_path; children.insert(child_path); } @@ -234,6 +235,13 @@ void KeeperStorage::Node::shallowCopy(const KeeperStorage::Node & other) cached_digest = other.cached_digest; } +void KeeperStorage::Node::recalculateSize() +{ + size_bytes = sizeof(Node); + size_bytes += children.size() * sizeof(decltype(children)::value_type); + size_bytes += data.size(); +} + KeeperStorage::KeeperStorage( int64_t tick_time_ms, const String & superdigest_, const KeeperContextPtr & keeper_context_, const bool initialize_system_nodes) : session_expiry_queue(tick_time_ms), keeper_context(keeper_context_), superdigest(superdigest_) @@ -2407,5 +2415,10 @@ uint64_t KeeperStorage::getTotalEphemeralNodesCount() const return ret; } +void KeeperStorage::recalculateStats() +{ + container.recalculateDataSize(); +} + } diff --git a/src/Coordination/KeeperStorage.h b/src/Coordination/KeeperStorage.h index a40cca8e778..be528072df4 100644 --- a/src/Coordination/KeeperStorage.h +++ b/src/Coordination/KeeperStorage.h @@ -47,7 +47,7 @@ public: const auto & getData() const noexcept { return data; } - void addChild(StringRef child_path); + void addChild(StringRef child_path, bool update_size = true); void removeChild(StringRef child_path); @@ -64,6 +64,8 @@ public: // (e.g. we don't need to copy list of children) void shallowCopy(const Node & other); + void recalculateSize(); + private: String data; ChildrenSet children{}; @@ -466,6 +468,7 @@ public: void dumpWatchesByPath(WriteBufferFromOwnString & buf) const; void dumpSessionsAndEphemerals(WriteBufferFromOwnString & buf) const; + void recalculateStats(); private: void removeDigest(const Node & node, std::string_view path); void addDigest(const Node & node, std::string_view path); diff --git a/src/Coordination/SnapshotableHashTable.h b/src/Coordination/SnapshotableHashTable.h index 27572ab86c7..cfa3098b4a1 100644 --- a/src/Coordination/SnapshotableHashTable.h +++ b/src/Coordination/SnapshotableHashTable.h @@ -64,7 +64,7 @@ private: /// value_size: size of value to add /// old_value_size: size of value to minus /// old_value_size=0 means there is no old value with the same key. - void updateDataSize(OperationType op_type, uint64_t key_size, uint64_t value_size, uint64_t old_value_size) + void updateDataSize(OperationType op_type, uint64_t key_size, uint64_t value_size, uint64_t old_value_size, bool remove_old = true) { switch (op_type) { @@ -94,14 +94,14 @@ private: case UPDATE_VALUE: approximate_data_size += key_size; approximate_data_size += value_size; - if (!snapshot_mode) + if (remove_old) { approximate_data_size -= key_size; approximate_data_size -= old_value_size; } break; case ERASE: - if (!snapshot_mode) + if (remove_old) { approximate_data_size -= key_size; approximate_data_size -= old_value_size; @@ -177,7 +177,7 @@ public: list_itr->value = value; } } - updateDataSize(INSERT_OR_REPLACE, key.size(), value.sizeInBytes(), old_value_size); + updateDataSize(INSERT_OR_REPLACE, key.size(), value.sizeInBytes(), old_value_size, !snapshot_mode); } bool erase(const std::string & key) @@ -202,7 +202,7 @@ public: list.erase(list_itr); } - updateDataSize(ERASE, key.size(), 0, old_data_size); + updateDataSize(ERASE, key.size(), 0, old_data_size, !snapshot_mode); return true; } @@ -222,6 +222,7 @@ public: const_iterator ret; + bool remove_old_size = true; if (snapshot_mode) { /// We in snapshot mode but updating some node which is already more @@ -237,6 +238,8 @@ public: auto itr = list.insert(list.end(), std::move(elem_copy)); it->getMapped() = itr; ret = itr; + + remove_old_size = false; } else { @@ -250,7 +253,7 @@ public: ret = list_itr; } - updateDataSize(UPDATE_VALUE, key.size, ret->value.sizeInBytes(), old_value_size); + updateDataSize(UPDATE_VALUE, key.size, ret->value.sizeInBytes(), old_value_size, remove_old_size); return ret; } @@ -319,6 +322,17 @@ public: return approximate_data_size; } + void recalculateDataSize() + { + approximate_data_size = 0; + for (auto & node : list) + { + node.value.recalculateSize(); + approximate_data_size += node.key.size; + approximate_data_size += node.value.sizeInBytes(); + } + } + uint64_t keyArenaSize() const { return arena.size(); diff --git a/src/Core/BackgroundSchedulePool.cpp b/src/Core/BackgroundSchedulePool.cpp index 993cfb6ef04..5384ee7f961 100644 --- a/src/Core/BackgroundSchedulePool.cpp +++ b/src/Core/BackgroundSchedulePool.cpp @@ -149,8 +149,9 @@ Coordination::WatchCallback BackgroundSchedulePoolTaskInfo::getWatchCallback() } -BackgroundSchedulePool::BackgroundSchedulePool(size_t size_, CurrentMetrics::Metric tasks_metric_, const char *thread_name_) +BackgroundSchedulePool::BackgroundSchedulePool(size_t size_, CurrentMetrics::Metric tasks_metric_, CurrentMetrics::Metric size_metric_, const char *thread_name_) : tasks_metric(tasks_metric_) + , size_metric(size_metric_, size_) , thread_name(thread_name_) { LOG_INFO(&Poco::Logger::get("BackgroundSchedulePool/" + thread_name), "Create BackgroundSchedulePool with {} threads", size_); @@ -177,6 +178,8 @@ void BackgroundSchedulePool::increaseThreadsCount(size_t new_threads_count) threads.resize(new_threads_count); for (size_t i = old_threads_count; i < new_threads_count; ++i) threads[i] = ThreadFromGlobalPoolNoTracingContextPropagation([this] { threadFunction(); }); + + size_metric.changeTo(new_threads_count); } diff --git a/src/Core/BackgroundSchedulePool.h b/src/Core/BackgroundSchedulePool.h index 0fb70b1f715..ef6fbfa68e9 100644 --- a/src/Core/BackgroundSchedulePool.h +++ b/src/Core/BackgroundSchedulePool.h @@ -54,7 +54,7 @@ public: void increaseThreadsCount(size_t new_threads_count); /// thread_name_ cannot be longer then 13 bytes (2 bytes is reserved for "/D" suffix for delayExecutionThreadFunction()) - BackgroundSchedulePool(size_t size_, CurrentMetrics::Metric tasks_metric_, const char *thread_name_); + BackgroundSchedulePool(size_t size_, CurrentMetrics::Metric tasks_metric_, CurrentMetrics::Metric size_metric_, const char *thread_name_); ~BackgroundSchedulePool(); private: @@ -91,6 +91,7 @@ private: DelayedTasks delayed_tasks; CurrentMetrics::Metric tasks_metric; + CurrentMetrics::Increment size_metric; std::string thread_name; }; diff --git a/src/Core/ExternalTable.cpp b/src/Core/ExternalTable.cpp index 217f8808185..36dcc677589 100644 --- a/src/Core/ExternalTable.cpp +++ b/src/Core/ExternalTable.cpp @@ -34,7 +34,7 @@ ExternalTableDataPtr BaseExternalTable::getData(ContextPtr context) { initReadBuffer(); initSampleBlock(); - auto input = context->getInputFormat(format, *read_buffer, sample_block, DEFAULT_BLOCK_SIZE); + auto input = context->getInputFormat(format, *read_buffer, sample_block, context->getSettingsRef().get("max_block_size").get()); auto data = std::make_unique(); data->pipe = std::make_unique(); @@ -135,7 +135,9 @@ void ExternalTablesHandler::handlePart(const Poco::Net::MessageHeader & header, if (settings.http_max_multipart_form_data_size) read_buffer = std::make_unique( stream, settings.http_max_multipart_form_data_size, - true, "the maximum size of multipart/form-data. This limit can be tuned by 'http_max_multipart_form_data_size' setting"); + /* trow_exception */ true, /* exact_limit */ std::optional(), + "the maximum size of multipart/form-data. " + "This limit can be tuned by 'http_max_multipart_form_data_size' setting"); else read_buffer = wrapReadBufferReference(stream); diff --git a/src/Core/MySQL/IMySQLReadPacket.cpp b/src/Core/MySQL/IMySQLReadPacket.cpp index 74f309d0294..39b2e5bbfb5 100644 --- a/src/Core/MySQL/IMySQLReadPacket.cpp +++ b/src/Core/MySQL/IMySQLReadPacket.cpp @@ -33,13 +33,13 @@ void IMySQLReadPacket::readPayloadWithUnpacked(ReadBuffer & in) void LimitedReadPacket::readPayload(ReadBuffer &in, uint8_t &sequence_id) { - LimitReadBuffer limited(in, 10000, true, "too long MySQL packet."); + LimitReadBuffer limited(in, 10000, /* trow_exception */ true, /* exact_limit */ {}, "too long MySQL packet."); IMySQLReadPacket::readPayload(limited, sequence_id); } void LimitedReadPacket::readPayloadWithUnpacked(ReadBuffer & in) { - LimitReadBuffer limited(in, 10000, true, "too long MySQL packet."); + LimitReadBuffer limited(in, 10000, /* trow_exception */ true, /* exact_limit */ {}, "too long MySQL packet."); IMySQLReadPacket::readPayloadWithUnpacked(limited); } diff --git a/src/Core/ProtocolDefines.h b/src/Core/ProtocolDefines.h index 3bbfb95f020..f687145668a 100644 --- a/src/Core/ProtocolDefines.h +++ b/src/Core/ProtocolDefines.h @@ -35,7 +35,6 @@ #define DBMS_MERGE_TREE_PART_INFO_VERSION 1 -/// Minimum revision supporting interserver secret. #define DBMS_MIN_REVISION_WITH_INTERSERVER_SECRET 54441 #define DBMS_MIN_REVISION_WITH_X_FORWARDED_FOR_IN_CLIENT_INFO 54443 @@ -54,7 +53,7 @@ /// NOTE: DBMS_TCP_PROTOCOL_VERSION has nothing common with VERSION_REVISION, /// later is just a number for server version (one number instead of commit SHA) /// for simplicity (sometimes it may be more convenient in some use cases). -#define DBMS_TCP_PROTOCOL_VERSION 54461 +#define DBMS_TCP_PROTOCOL_VERSION 54462 #define DBMS_MIN_PROTOCOL_VERSION_WITH_INITIAL_QUERY_START_TIME 54449 @@ -72,3 +71,5 @@ #define DBMS_MIN_PROTOCOL_VERSION_WITH_SERVER_QUERY_TIME_IN_PROGRESS 54460 #define DBMS_MIN_PROTOCOL_VERSION_WITH_PASSWORD_COMPLEXITY_RULES 54461 + +#define DBMS_MIN_REVISION_WITH_INTERSERVER_SECRET_V2 54462 diff --git a/src/Core/ServerSettings.h b/src/Core/ServerSettings.h index 1e884266c17..abc5b4d14d8 100644 --- a/src/Core/ServerSettings.h +++ b/src/Core/ServerSettings.h @@ -21,6 +21,9 @@ namespace DB M(UInt64, max_io_thread_pool_size, 100, "The maximum number of threads that would be used for IO operations", 0) \ M(UInt64, max_io_thread_pool_free_size, 0, "Max free size for IO thread pool.", 0) \ M(UInt64, io_thread_pool_queue_size, 10000, "Queue size for IO thread pool.", 0) \ + M(UInt64, max_backups_io_thread_pool_size, 1000, "The maximum number of threads that would be used for IO operations for BACKUP queries", 0) \ + M(UInt64, max_backups_io_thread_pool_free_size, 0, "Max free size for backups IO thread pool.", 0) \ + M(UInt64, backups_io_thread_pool_queue_size, 0, "Queue size for backups IO thread pool.", 0) \ M(Int32, max_connections, 1024, "Max server connections.", 0) \ M(UInt32, asynchronous_metrics_update_period_s, 1, "Period in seconds for updating asynchronous metrics.", 0) \ M(UInt32, asynchronous_heavy_metrics_update_period_s, 120, "Period in seconds for updating asynchronous metrics.", 0) \ @@ -75,4 +78,3 @@ struct ServerSettings : public BaseSettings }; } - diff --git a/src/Core/Settings.h b/src/Core/Settings.h index a65f2ccb60f..755e52adb06 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -50,14 +50,13 @@ class IColumn; M(UInt64, max_download_buffer_size, 10*1024*1024, "The maximal size of buffer for parallel downloading (e.g. for URL engine) per each thread.", 0) \ M(UInt64, max_read_buffer_size, DBMS_DEFAULT_BUFFER_SIZE, "The maximum size of the buffer to read from the filesystem.", 0) \ M(UInt64, max_distributed_connections, 1024, "The maximum number of connections for distributed processing of one query (should be greater than max_threads).", 0) \ - M(UInt64, max_query_size, DBMS_DEFAULT_MAX_QUERY_SIZE, "Which part of the query can be read into RAM for parsing (the remaining data for INSERT, if any, is read later)", 0) \ + M(UInt64, max_query_size, DBMS_DEFAULT_MAX_QUERY_SIZE, "The maximum number of bytes of a query string parsed by the SQL parser. Data in the VALUES clause of INSERT queries is processed by a separate stream parser (that consumes O(1) RAM) and not affected by this restriction.", 0) \ M(UInt64, interactive_delay, 100000, "The interval in microseconds to check if the request is cancelled, and to send progress info.", 0) \ M(Seconds, connect_timeout, DBMS_DEFAULT_CONNECT_TIMEOUT_SEC, "Connection timeout if there are no replicas.", 0) \ M(Milliseconds, connect_timeout_with_failover_ms, 50, "Connection timeout for selecting first healthy replica.", 0) \ M(Milliseconds, connect_timeout_with_failover_secure_ms, 100, "Connection timeout for selecting first healthy replica (for secure connections).", 0) \ M(Seconds, receive_timeout, DBMS_DEFAULT_RECEIVE_TIMEOUT_SEC, "Timeout for receiving data from network, in seconds. If no bytes were received in this interval, exception is thrown. If you set this setting on client, the 'send_timeout' for the socket will be also set on the corresponding connection end on the server.", 0) \ M(Seconds, send_timeout, DBMS_DEFAULT_SEND_TIMEOUT_SEC, "Timeout for sending data to network, in seconds. If client needs to sent some data, but it did not able to send any bytes in this interval, exception is thrown. If you set this setting on client, the 'receive_timeout' for the socket will be also set on the corresponding connection end on the server.", 0) \ - M(Seconds, drain_timeout, 3, "Timeout for draining remote connections, -1 means synchronous drain without ignoring errors", 0) \ M(Seconds, tcp_keep_alive_timeout, 290 /* less than DBMS_DEFAULT_RECEIVE_TIMEOUT_SEC */, "The time in seconds the connection needs to remain idle before TCP starts sending keepalive probes", 0) \ M(Milliseconds, hedged_connection_timeout_ms, 100, "Connection timeout for establishing connection with replica for Hedged requests", 0) \ M(Milliseconds, receive_data_timeout_ms, 2000, "Connection timeout for receiving first packet of data or packet with positive progress from replica", 0) \ @@ -147,6 +146,8 @@ class IColumn; M(UInt64, max_parallel_replicas, 1, "The maximum number of replicas of each shard used when the query is executed. For consistency (to get different parts of the same partition), this option only works for the specified sampling key. The lag of the replicas is not controlled.", 0) \ M(UInt64, parallel_replicas_count, 0, "This is internal setting that should not be used directly and represents an implementation detail of the 'parallel replicas' mode. This setting will be automatically set up by the initiator server for distributed queries to the number of parallel replicas participating in query processing.", 0) \ M(UInt64, parallel_replica_offset, 0, "This is internal setting that should not be used directly and represents an implementation detail of the 'parallel replicas' mode. This setting will be automatically set up by the initiator server for distributed queries to the index of the replica participating in query processing among parallel replicas.", 0) \ + M(String, parallel_replicas_custom_key, "", "Custom key assigning work to replicas when parallel replicas are used.", 0) \ + M(ParallelReplicasCustomKeyFilterType, parallel_replicas_custom_key_filter_type, ParallelReplicasCustomKeyFilterType::DEFAULT, "Type of filter to use with custom key for parallel replicas. default - use modulo operation on the custom key, range - use range filter on custom key using all possible values for the value type of custom key.", 0) \ \ M(String, cluster_for_parallel_replicas, "default", "Cluster for a shard in which current server is located", 0) \ M(Bool, allow_experimental_parallel_reading_from_replicas, false, "If true, ClickHouse will send a SELECT query to all replicas of a table. It will work for any kind on MergeTree table.", 0) \ @@ -252,6 +253,8 @@ class IColumn; M(Bool, send_progress_in_http_headers, false, "Send progress notifications using X-ClickHouse-Progress headers. Some clients do not support high amount of HTTP headers (Python requests in particular), so it is disabled by default.", 0) \ \ M(UInt64, http_headers_progress_interval_ms, 100, "Do not send HTTP headers X-ClickHouse-Progress more frequently than at each specified interval.", 0) \ + M(Bool, http_wait_end_of_query, false, "Enable HTTP response buffering on the server-side.", 0) \ + M(UInt64, http_response_buffer_size, false, "The number of bytes to buffer in the server memory before sending a HTTP response to the client or flushing to disk (when http_wait_end_of_query is enabled).", 0) \ \ M(Bool, fsync_metadata, true, "Do fsync after changing metadata for tables and databases (.sql files). Could be disabled in case of poor latency on server with high load of DDL queries and high load of disk subsystem.", 0) \ \ @@ -282,8 +285,6 @@ class IColumn; M(Milliseconds, sleep_in_send_data_ms, 0, "Time to sleep in sending data in TCPHandler", 0) \ M(Milliseconds, sleep_after_receiving_query_ms, 0, "Time to sleep after receiving query in TCPHandler", 0) \ M(UInt64, unknown_packet_in_send_data, 0, "Send unknown packet instead of data Nth data packet", 0) \ - /** Settings for testing connection collector */ \ - M(Milliseconds, sleep_in_receive_cancel_ms, 0, "Time to sleep in receiving cancel in TCPHandler", 0) \ \ M(Bool, insert_allow_materialized_columns, false, "If setting is enabled, Allow materialized columns in INSERT.", 0) \ M(Seconds, http_connection_timeout, DEFAULT_HTTP_READ_BUFFER_CONNECTION_TIMEOUT, "HTTP connection timeout.", 0) \ @@ -414,6 +415,10 @@ class IColumn; \ M(UInt64, backup_threads, 16, "The maximum number of threads to execute BACKUP requests.", 0) \ M(UInt64, restore_threads, 16, "The maximum number of threads to execute RESTORE requests.", 0) \ + M(UInt64, backup_keeper_max_retries, 20, "Max retries for keeper operations during backup", 0) \ + M(UInt64, backup_keeper_retry_initial_backoff_ms, 100, "Initial backoff timeout for [Zoo]Keeper operations during backup", 0) \ + M(UInt64, backup_keeper_retry_max_backoff_ms, 5000, "Max backoff timeout for [Zoo]Keeper operations during backup", 0) \ + M(UInt64, backup_batch_size_for_keeper_multiread, 10000, "Maximum size of batch for multiread request to [Zoo]Keeper during backup", 0) \ \ M(Bool, log_profile_events, true, "Log query performance statistics into the query_log, query_thread_log and query_views_log.", 0) \ M(Bool, log_query_settings, true, "Log query settings into the query_log.", 0) \ @@ -510,6 +515,7 @@ class IColumn; M(Bool, allow_experimental_alter_materialized_view_structure, false, "Allow atomic alter on Materialized views. Work in progress.", 0) \ M(Bool, enable_early_constant_folding, true, "Enable query optimization where we analyze function and subqueries results and rewrite query if there're constants there", 0) \ M(Bool, deduplicate_blocks_in_dependent_materialized_views, false, "Should deduplicate blocks for materialized views if the block is not a duplicate for the table. Use true to always deduplicate in dependent tables.", 0) \ + M(Bool, materialized_views_ignore_errors, false, "Allows to ignore errors for MATERIALIZED VIEW, and deliver original block to the table regardless of MVs", 0) \ M(Bool, use_compact_format_in_distributed_parts_names, true, "Changes format of directories names for distributed table insert parts.", 0) \ M(Bool, validate_polygons, true, "Throw exception if polygon is invalid in function pointInPolygon (e.g. self-tangent, self-intersecting). If the setting is false, the function will accept invalid polygons but may silently return wrong result.", 0) \ M(UInt64, max_parser_depth, DBMS_DEFAULT_MAX_PARSER_DEPTH, "Maximum parser depth (recursion depth of recursive descend parser).", 0) \ @@ -712,6 +718,7 @@ class IColumn; M(Float, insert_keeper_fault_injection_probability, 0.0f, "Approximate probability of failure for a keeper request during insert. Valid value is in interval [0.0f, 1.0f]", 0) \ M(UInt64, insert_keeper_fault_injection_seed, 0, "0 - random seed, otherwise the setting value", 0) \ M(Bool, force_aggregation_in_order, false, "Force use of aggregation in order on remote nodes during distributed aggregation. PLEASE, NEVER CHANGE THIS SETTING VALUE MANUALLY!", IMPORTANT) \ + M(UInt64, http_max_request_param_data_size, 10_MiB, "Limit on size of request data used as a query parameter in predefined HTTP requests.", 0) \ // End of COMMON_SETTINGS // Please add settings related to formats into the FORMAT_FACTORY_SETTINGS and move obsolete settings to OBSOLETE_SETTINGS. @@ -752,7 +759,7 @@ class IColumn; MAKE_OBSOLETE(M, Seconds, temporary_live_view_timeout, 1) \ MAKE_OBSOLETE(M, Milliseconds, async_insert_cleanup_timeout_ms, 1000) \ MAKE_OBSOLETE(M, Bool, optimize_fuse_sum_count_avg, 0) \ - + MAKE_OBSOLETE(M, Seconds, drain_timeout, 3) \ /** The section above is for obsolete settings. Do not add anything there. */ @@ -820,6 +827,8 @@ class IColumn; M(UInt64, input_format_csv_skip_first_lines, 0, "Skip specified number of lines at the beginning of data in CSV format", 0) \ M(UInt64, input_format_tsv_skip_first_lines, 0, "Skip specified number of lines at the beginning of data in TSV format", 0) \ \ + M(Bool, input_format_native_allow_types_conversion, true, "Allow data types conversion in Native input format", 0) \ + \ M(DateTimeInputFormat, date_time_input_format, FormatSettings::DateTimeInputFormat::Basic, "Method to read DateTime from text input formats. Possible values: 'basic', 'best_effort' and 'best_effort_us'.", 0) \ M(DateTimeOutputFormat, date_time_output_format, FormatSettings::DateTimeOutputFormat::Simple, "Method to write DateTime to text output. Possible values: 'simple', 'iso', 'unix_timestamp'.", 0) \ \ @@ -858,6 +867,7 @@ class IColumn; M(Bool, output_format_parquet_string_as_string, false, "Use Parquet String type instead of Binary for String columns.", 0) \ M(Bool, output_format_parquet_fixed_string_as_fixed_byte_array, true, "Use Parquet FIXED_LENGTH_BYTE_ARRAY type instead of Binary for FixedString columns.", 0) \ M(ParquetVersion, output_format_parquet_version, "2.latest", "Parquet format version for output format. Supported versions: 1.0, 2.4, 2.6 and 2.latest (default)", 0) \ + M(ParquetCompression, output_format_parquet_compression_method, "lz4", "Compression method for Parquet output format. Supported codecs: snappy, lz4, brotli, zstd, gzip, none (uncompressed)", 0) \ M(String, output_format_avro_codec, "", "Compression codec used for output. Possible values: 'null', 'deflate', 'snappy'.", 0) \ M(UInt64, output_format_avro_sync_interval, 16 * 1024, "Sync interval in bytes.", 0) \ M(String, output_format_avro_string_column_pattern, "", "For Avro format: regexp of String columns to select as AVRO string.", 0) \ @@ -900,8 +910,10 @@ class IColumn; M(Bool, output_format_arrow_low_cardinality_as_dictionary, false, "Enable output LowCardinality type as Dictionary Arrow type", 0) \ M(Bool, output_format_arrow_string_as_string, false, "Use Arrow String type instead of Binary for String columns", 0) \ M(Bool, output_format_arrow_fixed_string_as_fixed_byte_array, true, "Use Arrow FIXED_SIZE_BINARY type instead of Binary for FixedString columns.", 0) \ + M(ArrowCompression, output_format_arrow_compression_method, "lz4_frame", "Compression method for Arrow output format. Supported codecs: lz4_frame, zstd, none (uncompressed)", 0) \ \ M(Bool, output_format_orc_string_as_string, false, "Use ORC String type instead of Binary for String columns", 0) \ + M(ORCCompression, output_format_orc_compression_method, "lz4", "Compression method for ORC output format. Supported codecs: lz4, snappy, zlib, zstd, none (uncompressed)", 0) \ \ M(EnumComparingMode, format_capn_proto_enum_comparising_mode, FormatSettings::EnumComparingMode::BY_VALUES, "How to map ClickHouse Enum and CapnProto Enum", 0) \ \ diff --git a/src/Core/SettingsChangesHistory.h b/src/Core/SettingsChangesHistory.h index 04f328bb665..caf18cf8fb8 100644 --- a/src/Core/SettingsChangesHistory.h +++ b/src/Core/SettingsChangesHistory.h @@ -81,7 +81,11 @@ namespace SettingsChangesHistory static std::map settings_changes_history = { {"23.3", {{"output_format_parquet_version", "1.0", "2.latest", "Use latest Parquet format version for output format"}, - {"input_format_json_ignore_unknown_keys_in_named_tuple", false, true, "Improve parsing JSON objects as named tuples"}}}, + {"input_format_json_ignore_unknown_keys_in_named_tuple", false, true, "Improve parsing JSON objects as named tuples"}, + {"input_format_native_allow_types_conversion", false, true, "Allow types conversion in Native input forma"}, + {"output_format_arrow_compression_method", "none", "lz4_frame", "Use lz4 compression in Arrow output format by default"}, + {"output_format_parquet_compression_method", "snappy", "lz4", "Use lz4 compression in Parquet output format by default"}, + {"output_format_orc_compression_method", "none", "lz4_frame", "Use lz4 compression in ORC output format by default"}}}, {"23.2", {{"output_format_parquet_fixed_string_as_fixed_byte_array", false, true, "Use Parquet FIXED_LENGTH_BYTE_ARRAY type for FixedString by default"}, {"output_format_arrow_fixed_string_as_fixed_byte_array", false, true, "Use Arrow FIXED_SIZE_BINARY type for FixedString by default"}, {"query_plan_remove_redundant_distinct", false, true, "Remove redundant Distinct step in query plan"}, diff --git a/src/Core/SettingsEnums.cpp b/src/Core/SettingsEnums.cpp index 9e1ab585bb0..e0f16ea00db 100644 --- a/src/Core/SettingsEnums.cpp +++ b/src/Core/SettingsEnums.cpp @@ -158,7 +158,7 @@ IMPLEMENT_SETTING_ENUM(EscapingRule, ErrorCodes::BAD_ARGUMENTS, {"XML", FormatSettings::EscapingRule::XML}, {"Raw", FormatSettings::EscapingRule::Raw}}) -IMPLEMENT_SETTING_ENUM(MsgPackUUIDRepresentation , ErrorCodes::BAD_ARGUMENTS, +IMPLEMENT_SETTING_ENUM(MsgPackUUIDRepresentation, ErrorCodes::BAD_ARGUMENTS, {{"bin", FormatSettings::MsgPackUUIDRepresentation::BIN}, {"str", FormatSettings::MsgPackUUIDRepresentation::STR}, {"ext", FormatSettings::MsgPackUUIDRepresentation::EXT}}) @@ -167,16 +167,39 @@ IMPLEMENT_SETTING_ENUM(Dialect, ErrorCodes::BAD_ARGUMENTS, {{"clickhouse", Dialect::clickhouse}, {"kusto", Dialect::kusto}}) +IMPLEMENT_SETTING_ENUM(ParallelReplicasCustomKeyFilterType, ErrorCodes::BAD_ARGUMENTS, + {{"default", ParallelReplicasCustomKeyFilterType::DEFAULT}, + {"range", ParallelReplicasCustomKeyFilterType::RANGE}}) + IMPLEMENT_SETTING_ENUM(LocalFSReadMethod, ErrorCodes::BAD_ARGUMENTS, {{"mmap", LocalFSReadMethod::mmap}, {"pread", LocalFSReadMethod::pread}, {"read", LocalFSReadMethod::read}}) - IMPLEMENT_SETTING_ENUM_WITH_RENAME(ParquetVersion, ErrorCodes::BAD_ARGUMENTS, {{"1.0", FormatSettings::ParquetVersion::V1_0}, {"2.4", FormatSettings::ParquetVersion::V2_4}, {"2.6", FormatSettings::ParquetVersion::V2_6}, {"2.latest", FormatSettings::ParquetVersion::V2_LATEST}}) +IMPLEMENT_SETTING_ENUM(ParquetCompression, ErrorCodes::BAD_ARGUMENTS, + {{"none", FormatSettings::ParquetCompression::NONE}, + {"snappy", FormatSettings::ParquetCompression::SNAPPY}, + {"zstd", FormatSettings::ParquetCompression::ZSTD}, + {"gzip", FormatSettings::ParquetCompression::GZIP}, + {"lz4", FormatSettings::ParquetCompression::LZ4}, + {"brotli", FormatSettings::ParquetCompression::BROTLI}}) + +IMPLEMENT_SETTING_ENUM(ArrowCompression, ErrorCodes::BAD_ARGUMENTS, + {{"none", FormatSettings::ArrowCompression::NONE}, + {"lz4_frame", FormatSettings::ArrowCompression::LZ4_FRAME}, + {"zstd", FormatSettings::ArrowCompression::ZSTD}}) + +IMPLEMENT_SETTING_ENUM(ORCCompression, ErrorCodes::BAD_ARGUMENTS, + {{"none", FormatSettings::ORCCompression::NONE}, + {"snappy", FormatSettings::ORCCompression::SNAPPY}, + {"zstd", FormatSettings::ORCCompression::ZSTD}, + {"zlib", FormatSettings::ORCCompression::ZLIB}, + {"lz4", FormatSettings::ORCCompression::LZ4}}) + } diff --git a/src/Core/SettingsEnums.h b/src/Core/SettingsEnums.h index 139a04f3a5a..3ae7bfaa673 100644 --- a/src/Core/SettingsEnums.h +++ b/src/Core/SettingsEnums.h @@ -194,6 +194,12 @@ DECLARE_SETTING_ENUM_WITH_RENAME(EscapingRule, FormatSettings::EscapingRule) DECLARE_SETTING_ENUM_WITH_RENAME(MsgPackUUIDRepresentation, FormatSettings::MsgPackUUIDRepresentation) +DECLARE_SETTING_ENUM_WITH_RENAME(ParquetCompression, FormatSettings::ParquetCompression) + +DECLARE_SETTING_ENUM_WITH_RENAME(ArrowCompression, FormatSettings::ArrowCompression) + +DECLARE_SETTING_ENUM_WITH_RENAME(ORCCompression, FormatSettings::ORCCompression) + enum class Dialect { clickhouse, @@ -203,5 +209,13 @@ enum class Dialect DECLARE_SETTING_ENUM(Dialect) +enum class ParallelReplicasCustomKeyFilterType : uint8_t +{ + DEFAULT, + RANGE, +}; + +DECLARE_SETTING_ENUM(ParallelReplicasCustomKeyFilterType) + DECLARE_SETTING_ENUM(LocalFSReadMethod) } diff --git a/src/Daemon/BaseDaemon.cpp b/src/Daemon/BaseDaemon.cpp index 60179fd5317..18c4c0d97a0 100644 --- a/src/Daemon/BaseDaemon.cpp +++ b/src/Daemon/BaseDaemon.cpp @@ -134,7 +134,7 @@ static void terminateRequestedSignalHandler(int sig, siginfo_t *, void *) } -static std::atomic fatal_error_printed{false}; +static std::atomic_flag fatal_error_printed; /** Handler for "fault" or diagnostic signals. Send data about fault to separate thread to write into log. */ @@ -165,7 +165,7 @@ static void signalHandler(int sig, siginfo_t * info, void * context) for (size_t i = 0; i < 300; ++i) { /// We will synchronize with the thread printing the messages with an atomic variable to finish earlier. - if (fatal_error_printed) + if (fatal_error_printed.test()) break; /// This coarse method of synchronization is perfectly ok for fatal signals. @@ -374,7 +374,7 @@ private: } /// Write symbolized stack trace line by line for better grep-ability. - stack_trace.toStringEveryLine([&](const std::string & s) { LOG_FATAL(log, fmt::runtime(s)); }); + stack_trace.toStringEveryLine([&](std::string_view s) { LOG_FATAL(log, fmt::runtime(s)); }); #if defined(OS_LINUX) /// Write information about binary checksum. It can be difficult to calculate, so do it only after printing stack trace. @@ -421,7 +421,7 @@ private: if (thread_ptr) thread_ptr->onFatalError(); - fatal_error_printed = true; + fatal_error_printed.test_and_set(); } }; diff --git a/src/Daemon/SentryWriter.cpp b/src/Daemon/SentryWriter.cpp index 9f4f18e64d1..3c62e54b117 100644 --- a/src/Daemon/SentryWriter.cpp +++ b/src/Daemon/SentryWriter.cpp @@ -13,7 +13,7 @@ #include #include #include -#include +#include #include "config.h" #include "config_version.h" diff --git a/src/DataTypes/DataTypeLowCardinality.cpp b/src/DataTypes/DataTypeLowCardinality.cpp index 5e3a1cd3a0e..8293455cabc 100644 --- a/src/DataTypes/DataTypeLowCardinality.cpp +++ b/src/DataTypes/DataTypeLowCardinality.cpp @@ -55,7 +55,7 @@ namespace } template - void operator()(Id) + void operator()(TypeList) { if (typeid_cast *>(&keys_type)) column = creator(static_cast *>(nullptr)); diff --git a/src/DataTypes/DataTypeLowCardinality.h b/src/DataTypes/DataTypeLowCardinality.h index 57f67ddad7a..d301a0f5443 100644 --- a/src/DataTypes/DataTypeLowCardinality.h +++ b/src/DataTypes/DataTypeLowCardinality.h @@ -86,6 +86,6 @@ DataTypePtr recursiveRemoveLowCardinality(const DataTypePtr & type); ColumnPtr recursiveRemoveLowCardinality(const ColumnPtr & column); /// Convert column of type from_type to type to_type by converting nested LowCardinality columns. -ColumnPtr recursiveTypeConversion(const ColumnPtr & column, const DataTypePtr & from_type, const DataTypePtr & to_type); +ColumnPtr recursiveLowCardinalityTypeConversion(const ColumnPtr & column, const DataTypePtr & from_type, const DataTypePtr & to_type); } diff --git a/src/DataTypes/DataTypeLowCardinalityHelpers.cpp b/src/DataTypes/DataTypeLowCardinalityHelpers.cpp index 8a61afee420..98eb76267a4 100644 --- a/src/DataTypes/DataTypeLowCardinalityHelpers.cpp +++ b/src/DataTypes/DataTypeLowCardinalityHelpers.cpp @@ -113,7 +113,7 @@ ColumnPtr recursiveRemoveLowCardinality(const ColumnPtr & column) return column; } -ColumnPtr recursiveTypeConversion(const ColumnPtr & column, const DataTypePtr & from_type, const DataTypePtr & to_type) +ColumnPtr recursiveLowCardinalityTypeConversion(const ColumnPtr & column, const DataTypePtr & from_type, const DataTypePtr & to_type) { if (!column) return column; @@ -128,7 +128,7 @@ ColumnPtr recursiveTypeConversion(const ColumnPtr & column, const DataTypePtr & if (const auto * column_const = typeid_cast(column.get())) { const auto & nested = column_const->getDataColumnPtr(); - auto nested_no_lc = recursiveTypeConversion(nested, from_type, to_type); + auto nested_no_lc = recursiveLowCardinalityTypeConversion(nested, from_type, to_type); if (nested.get() == nested_no_lc.get()) return column; @@ -164,7 +164,7 @@ ColumnPtr recursiveTypeConversion(const ColumnPtr & column, const DataTypePtr & const auto & nested_to = to_array_type->getNestedType(); return ColumnArray::create( - recursiveTypeConversion(column_array->getDataPtr(), nested_from, nested_to), + recursiveLowCardinalityTypeConversion(column_array->getDataPtr(), nested_from, nested_to), column_array->getOffsetsPtr()); } } @@ -187,7 +187,7 @@ ColumnPtr recursiveTypeConversion(const ColumnPtr & column, const DataTypePtr & for (size_t i = 0; i < columns.size(); ++i) { auto & element = columns[i]; - auto element_no_lc = recursiveTypeConversion(element, from_elements.at(i), to_elements.at(i)); + auto element_no_lc = recursiveLowCardinalityTypeConversion(element, from_elements.at(i), to_elements.at(i)); if (element.get() != element_no_lc.get()) { element = element_no_lc; diff --git a/src/DataTypes/Serializations/SerializationInfo.cpp b/src/DataTypes/Serializations/SerializationInfo.cpp index 313fa1fa235..4e5790ad58d 100644 --- a/src/DataTypes/Serializations/SerializationInfo.cpp +++ b/src/DataTypes/Serializations/SerializationInfo.cpp @@ -97,6 +97,41 @@ MutableSerializationInfoPtr SerializationInfo::clone() const return std::make_shared(kind, settings, data); } +/// Returns true if all rows with default values of type 'lhs' +/// are mapped to default values of type 'rhs' after conversion. +static bool preserveDefaultsAfterConversion(const IDataType & lhs, const IDataType & rhs) +{ + if (lhs.equals(rhs)) + return true; + + bool lhs_is_columned_as_numeric = isColumnedAsNumber(lhs) || isColumnedAsDecimal(lhs); + bool rhs_is_columned_as_numeric = isColumnedAsNumber(rhs) || isColumnedAsDecimal(rhs); + + if (lhs_is_columned_as_numeric && rhs_is_columned_as_numeric) + return true; + + if (isStringOrFixedString(lhs) && isStringOrFixedString(rhs)) + return true; + + return false; +} + +std::shared_ptr SerializationInfo::createWithType( + const IDataType & old_type, + const IDataType & new_type, + const Settings & new_settings) const +{ + auto new_kind = kind; + if (new_kind == ISerialization::Kind::SPARSE) + { + if (!new_type.supportsSparseSerialization() + || !preserveDefaultsAfterConversion(old_type, new_type)) + new_kind = ISerialization::Kind::DEFAULT; + } + + return std::make_shared(new_kind, new_settings); +} + void SerializationInfo::serialializeKindBinary(WriteBuffer & out) const { writeBinary(static_cast(kind), out); diff --git a/src/DataTypes/Serializations/SerializationInfo.h b/src/DataTypes/Serializations/SerializationInfo.h index a4a5685253f..5b802b379e1 100644 --- a/src/DataTypes/Serializations/SerializationInfo.h +++ b/src/DataTypes/Serializations/SerializationInfo.h @@ -8,6 +8,7 @@ namespace DB { +class ReadBuffer; class ReadBuffer; class WriteBuffer; class NamesAndTypesList; @@ -51,6 +52,7 @@ public: virtual ~SerializationInfo() = default; virtual bool hasCustomSerialization() const { return kind != ISerialization::Kind::DEFAULT; } + virtual bool structureEquals(const SerializationInfo & rhs) const { return typeid(SerializationInfo) == typeid(rhs); } virtual void add(const IColumn & column); virtual void add(const SerializationInfo & other); @@ -59,6 +61,11 @@ public: virtual std::shared_ptr clone() const; + virtual std::shared_ptr createWithType( + const IDataType & old_type, + const IDataType & new_type, + const Settings & new_settings) const; + virtual void serialializeKindBinary(WriteBuffer & out) const; virtual void deserializeFromKindsBinary(ReadBuffer & in); @@ -85,7 +92,8 @@ using MutableSerializationInfoPtr = std::shared_ptr; using SerializationInfos = std::vector; using MutableSerializationInfos = std::vector; -class SerializationInfoByName : public std::unordered_map +/// The order is important because info is serialized to part metadata. +class SerializationInfoByName : public std::map { public: SerializationInfoByName() = default; diff --git a/src/DataTypes/Serializations/SerializationInfoTuple.cpp b/src/DataTypes/Serializations/SerializationInfoTuple.cpp index 6c326743e8a..d36668f03b6 100644 --- a/src/DataTypes/Serializations/SerializationInfoTuple.cpp +++ b/src/DataTypes/Serializations/SerializationInfoTuple.cpp @@ -28,6 +28,19 @@ bool SerializationInfoTuple::hasCustomSerialization() const return std::any_of(elems.begin(), elems.end(), [](const auto & elem) { return elem->hasCustomSerialization(); }); } +bool SerializationInfoTuple::structureEquals(const SerializationInfo & rhs) const +{ + const auto * rhs_tuple = typeid_cast(&rhs); + if (!rhs_tuple || elems.size() != rhs_tuple->elems.size()) + return false; + + for (size_t i = 0; i < elems.size(); ++i) + if (!elems[i]->structureEquals(*rhs_tuple->elems[i])) + return false; + + return true; +} + void SerializationInfoTuple::add(const IColumn & column) { SerializationInfo::add(column); @@ -84,6 +97,28 @@ MutableSerializationInfoPtr SerializationInfoTuple::clone() const return std::make_shared(std::move(elems_cloned), names, settings); } +MutableSerializationInfoPtr SerializationInfoTuple::createWithType( + const IDataType & old_type, + const IDataType & new_type, + const Settings & new_settings) const +{ + const auto & old_tuple = assert_cast(old_type); + const auto & new_tuple = assert_cast(new_type); + + const auto & old_elements = old_tuple.getElements(); + const auto & new_elements = new_tuple.getElements(); + + assert(elems.size() == old_elements.size()); + assert(elems.size() == new_elements.size()); + + MutableSerializationInfos infos; + infos.reserve(elems.size()); + for (size_t i = 0; i < elems.size(); ++i) + infos.push_back(elems[i]->createWithType(*old_elements[i], *new_elements[i], new_settings)); + + return std::make_shared(std::move(infos), names, new_settings); +} + void SerializationInfoTuple::serialializeKindBinary(WriteBuffer & out) const { SerializationInfo::serialializeKindBinary(out); diff --git a/src/DataTypes/Serializations/SerializationInfoTuple.h b/src/DataTypes/Serializations/SerializationInfoTuple.h index b01c629d2ff..a9f3bdb6c6e 100644 --- a/src/DataTypes/Serializations/SerializationInfoTuple.h +++ b/src/DataTypes/Serializations/SerializationInfoTuple.h @@ -11,6 +11,7 @@ public: SerializationInfoTuple(MutableSerializationInfos elems_, Names names_, const Settings & settings_); bool hasCustomSerialization() const override; + bool structureEquals(const SerializationInfo & rhs) const override; void add(const IColumn & column) override; void add(const SerializationInfo & other) override; @@ -19,6 +20,11 @@ public: MutableSerializationInfoPtr clone() const override; + MutableSerializationInfoPtr createWithType( + const IDataType & old_type, + const IDataType & new_type, + const Settings & new_settings) const override; + void serialializeKindBinary(WriteBuffer & out) const override; void deserializeFromKindsBinary(ReadBuffer & in) override; diff --git a/src/Databases/DatabaseAtomic.cpp b/src/Databases/DatabaseAtomic.cpp index 34c4fd3d5d8..7e20b6f6535 100644 --- a/src/Databases/DatabaseAtomic.cpp +++ b/src/Databases/DatabaseAtomic.cpp @@ -273,7 +273,7 @@ void DatabaseAtomic::renameTable(ContextPtr local_context, const String & table_ else renameNoReplace(old_metadata_path, new_metadata_path); - /// After metadata was successfully moved, the following methods should not throw (if them do, it's a logical error) + /// After metadata was successfully moved, the following methods should not throw (if they do, it's a logical error) table_data_path = detach(*this, table_name, table->storesDataOnDisk()); if (exchange) other_table_data_path = detach(other_db, to_table_name, other_table->storesDataOnDisk()); @@ -509,6 +509,9 @@ void DatabaseAtomic::tryCreateMetadataSymlink() { try { + /// fs::exists could return false for broken symlink + if (FS::isSymlinkNoThrow(metadata_symlink)) + fs::remove(metadata_symlink); fs::create_directory_symlink(metadata_path, path_to_metadata_symlink); } catch (...) diff --git a/src/Databases/DatabaseMemory.cpp b/src/Databases/DatabaseMemory.cpp index fda0bbe8032..3ede69d5362 100644 --- a/src/Databases/DatabaseMemory.cpp +++ b/src/Databases/DatabaseMemory.cpp @@ -26,7 +26,12 @@ namespace ErrorCodes DatabaseMemory::DatabaseMemory(const String & name_, ContextPtr context_) : DatabaseWithOwnTablesBase(name_, "DatabaseMemory(" + name_ + ")", context_) , data_path("data/" + escapeForFileName(database_name) + "/") -{} +{ + /// Temporary database should not have any data on the moment of its creation + /// In case of sudden server shutdown remove database folder of temporary database + if (name_ == DatabaseCatalog::TEMPORARY_DATABASE) + removeDataPath(context_); +} void DatabaseMemory::createTable( ContextPtr /*context*/, @@ -71,8 +76,7 @@ void DatabaseMemory::dropTable( if (table->storesDataOnDisk()) { - assert(getDatabaseName() != DatabaseCatalog::TEMPORARY_DATABASE); - fs::path table_data_dir{getTableDataPath(table_name)}; + fs::path table_data_dir{fs::path{getContext()->getPath()} / getTableDataPath(table_name)}; if (fs::exists(table_data_dir)) fs::remove_all(table_data_dir); } @@ -80,7 +84,6 @@ void DatabaseMemory::dropTable( catch (...) { std::lock_guard lock{mutex}; - assert(database_name != DatabaseCatalog::TEMPORARY_DATABASE); attachTableUnlocked(table_name, table); throw; } @@ -129,10 +132,15 @@ UUID DatabaseMemory::tryGetTableUUID(const String & table_name) const return UUIDHelpers::Nil; } +void DatabaseMemory::removeDataPath(ContextPtr local_context) +{ + std::filesystem::remove_all(local_context->getPath() + data_path); +} + void DatabaseMemory::drop(ContextPtr local_context) { /// Remove data on explicit DROP DATABASE - std::filesystem::remove_all(local_context->getPath() + data_path); + removeDataPath(local_context); } void DatabaseMemory::alterTable(ContextPtr local_context, const StorageID & table_id, const StorageInMemoryMetadata & metadata) diff --git a/src/Databases/DatabaseMemory.h b/src/Databases/DatabaseMemory.h index 6262543b0c1..0f703a0b46e 100644 --- a/src/Databases/DatabaseMemory.h +++ b/src/Databases/DatabaseMemory.h @@ -53,6 +53,8 @@ public: std::vector> getTablesForBackup(const FilterByNameFunction & filter, const ContextPtr & local_context) const override; private: + void removeDataPath(ContextPtr local_context); + const String data_path; using NameToASTCreate = std::unordered_map; NameToASTCreate create_queries TSA_GUARDED_BY(mutex); diff --git a/src/Dictionaries/MongoDBDictionarySource.cpp b/src/Dictionaries/MongoDBDictionarySource.cpp index bec566c29a6..a9555a94304 100644 --- a/src/Dictionaries/MongoDBDictionarySource.cpp +++ b/src/Dictionaries/MongoDBDictionarySource.cpp @@ -114,7 +114,11 @@ MongoDBDictionarySource::MongoDBDictionarySource( { if (!uri.empty()) { - Poco::URI poco_uri(uri); + // Connect with URI. + Poco::MongoDB::Connection::SocketFactory socket_factory; + connection->connect(uri, socket_factory); + + Poco::URI poco_uri(connection->uri()); // Parse database from URI. This is required for correctness -- the // cursor is created using database name and collection name, so we have @@ -134,10 +138,6 @@ MongoDBDictionarySource::MongoDBDictionarySource( { user.resize(separator); } - - // Connect with URI. - Poco::MongoDB::Connection::SocketFactory socket_factory; - connection->connect(uri, socket_factory); } else { diff --git a/src/Disks/FakeDiskTransaction.h b/src/Disks/FakeDiskTransaction.h index 46be885739e..5dae17041e1 100644 --- a/src/Disks/FakeDiskTransaction.h +++ b/src/Disks/FakeDiskTransaction.h @@ -68,6 +68,15 @@ public: return disk.writeFile(path, buf_size, mode, settings); } + void writeFileUsingCustomWriteObject( + const String & path, + WriteMode mode, + std::function & object_attributes)> + custom_write_object_function) override + { + disk.writeFileUsingCustomWriteObject(path, mode, std::move(custom_write_object_function)); + } + void removeFile(const std::string & path) override { disk.removeFile(path); diff --git a/src/Disks/IDisk.cpp b/src/Disks/IDisk.cpp index 2a60f32929c..4969cc7c700 100644 --- a/src/Disks/IDisk.cpp +++ b/src/Disks/IDisk.cpp @@ -38,6 +38,15 @@ void IDisk::copyFile(const String & from_file_path, IDisk & to_disk, const Strin out->finalize(); } +void IDisk::writeFileUsingCustomWriteObject( + const String &, WriteMode, std::function &)>) +{ + throw Exception( + ErrorCodes::NOT_IMPLEMENTED, + "Method `writeFileUsingCustomWriteObject()` is not implemented for disk: {}", + getDataSourceDescription().type); +} + DiskTransactionPtr IDisk::createTransaction() { diff --git a/src/Disks/IDisk.h b/src/Disks/IDisk.h index 79b2fefe964..ea117c0e0c6 100644 --- a/src/Disks/IDisk.h +++ b/src/Disks/IDisk.h @@ -209,6 +209,15 @@ public: WriteMode mode = WriteMode::Rewrite, const WriteSettings & settings = {}) = 0; + /// Write a file using a custom function to write an object to the disk's object storage. + /// This method is alternative to writeFile(), the difference is that writeFile() calls IObjectStorage::writeObject() + /// to write an object to the object storage while this method allows to specify a callback for that. + virtual void writeFileUsingCustomWriteObject( + const String & path, + WriteMode mode, + std::function & object_attributes)> + custom_write_object_function); + /// Remove file. Throws exception if file doesn't exists or it's a directory. /// Return whether file was finally removed. (For remote disks it is not always removed). virtual void removeFile(const String & path) = 0; diff --git a/src/Disks/IDiskTransaction.h b/src/Disks/IDiskTransaction.h index 02c8731428d..2edbe858c06 100644 --- a/src/Disks/IDiskTransaction.h +++ b/src/Disks/IDiskTransaction.h @@ -68,6 +68,13 @@ public: const WriteSettings & settings = {}, bool autocommit = true) = 0; + /// Write a file using a custom function to write an object to the disk's object storage. + virtual void writeFileUsingCustomWriteObject( + const String & path, + WriteMode mode, + std::function & object_attributes)> + custom_write_object_function) = 0; + /// Remove file. Throws exception if file doesn't exists or it's a directory. virtual void removeFile(const std::string & path) = 0; diff --git a/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp b/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp index 44a719c82bb..72346787cfb 100644 --- a/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp +++ b/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp @@ -4,7 +4,7 @@ #include #include #include -#include +#include #include #include diff --git a/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp b/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp index 43b6544acb0..8450e740ab5 100644 --- a/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp +++ b/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp @@ -5,7 +5,7 @@ #include #include #include -#include +#include #include #include diff --git a/src/Disks/ObjectStorages/DiskObjectStorage.cpp b/src/Disks/ObjectStorages/DiskObjectStorage.cpp index d55b1c91c07..44cb80558af 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorage.cpp +++ b/src/Disks/ObjectStorages/DiskObjectStorage.cpp @@ -577,6 +577,17 @@ std::unique_ptr DiskObjectStorage::writeFile( return result; } +void DiskObjectStorage::writeFileUsingCustomWriteObject( + const String & path, + WriteMode mode, + std::function & object_attributes)> + custom_write_object_function) +{ + LOG_TEST(log, "Write file: {}", path); + auto transaction = createObjectStorageTransaction(); + return transaction->writeFileUsingCustomWriteObject(path, mode, std::move(custom_write_object_function)); +} + void DiskObjectStorage::applyNewSettings( const Poco::Util::AbstractConfiguration & config, ContextPtr context_, const String &, const DisksMap &) { diff --git a/src/Disks/ObjectStorages/DiskObjectStorage.h b/src/Disks/ObjectStorages/DiskObjectStorage.h index a24acc270c0..d6723d1eb71 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorage.h +++ b/src/Disks/ObjectStorages/DiskObjectStorage.h @@ -152,6 +152,12 @@ public: WriteMode mode, const WriteSettings & settings) override; + void writeFileUsingCustomWriteObject( + const String & path, + WriteMode mode, + std::function & object_attributes)> + custom_write_object_function) override; + void copy(const String & from_path, const std::shared_ptr & to_disk, const String & to_path) override; void applyNewSettings(const Poco::Util::AbstractConfiguration & config, ContextPtr context_, const String &, const DisksMap &) override; diff --git a/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp b/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp index a9d82a3e0b1..072e747aa4a 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp +++ b/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp @@ -670,6 +670,44 @@ std::unique_ptr DiskObjectStorageTransaction::writeFile } +void DiskObjectStorageTransaction::writeFileUsingCustomWriteObject( + const String & path, + WriteMode mode, + std::function & object_attributes)> + custom_write_object_function) +{ + /// This function is a simplified and adapted version of DiskObjectStorageTransaction::writeFile(). + auto blob_name = object_storage.generateBlobNameForPath(path); + std::optional object_attributes; + + if (metadata_helper) + { + auto revision = metadata_helper->revision_counter + 1; + metadata_helper->revision_counter++; + object_attributes = { + {"path", path} + }; + blob_name = "r" + revisionToString(revision) + "-file-" + blob_name; + } + + auto object = StoredObject::create(object_storage, fs::path(metadata_storage.getObjectStorageRootPath()) / blob_name); + auto write_operation = std::make_unique(object_storage, metadata_storage, object); + + operations_to_execute.emplace_back(std::move(write_operation)); + + /// We always use mode Rewrite because we simulate append using metadata and different files + size_t object_size = std::move(custom_write_object_function)(object, WriteMode::Rewrite, object_attributes); + + /// Create metadata (see create_metadata_callback in DiskObjectStorageTransaction::writeFile()). + if (mode == WriteMode::Rewrite) + metadata_transaction->createMetadataFile(path, blob_name, object_size); + else + metadata_transaction->addBlobToMetadata(path, blob_name, object_size); + + metadata_transaction->commit(); +} + + void DiskObjectStorageTransaction::createHardLink(const std::string & src_path, const std::string & dst_path) { operations_to_execute.emplace_back( diff --git a/src/Disks/ObjectStorages/DiskObjectStorageTransaction.h b/src/Disks/ObjectStorages/DiskObjectStorageTransaction.h index 9e6bd5b6307..080a3e42057 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorageTransaction.h +++ b/src/Disks/ObjectStorages/DiskObjectStorageTransaction.h @@ -99,6 +99,13 @@ public: const WriteSettings & settings = {}, bool autocommit = true) override; + /// Write a file using a custom function to write an object to the disk's object storage. + void writeFileUsingCustomWriteObject( + const String & path, + WriteMode mode, + std::function & object_attributes)> + custom_write_object_function) override; + void removeFile(const std::string & path) override; void removeFileIfExists(const std::string & path) override; void removeDirectory(const std::string & path) override; diff --git a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp index cbba5ed64f9..83c0c7446a8 100644 --- a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp +++ b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp @@ -1,7 +1,4 @@ #include -#include -#include - #if USE_AWS_S3 @@ -18,10 +15,12 @@ #include #include #include +#include #include #include #include +#include #include #include #include diff --git a/src/Disks/ObjectStorages/S3/copyS3FileToDisk.cpp b/src/Disks/ObjectStorages/S3/copyS3FileToDisk.cpp new file mode 100644 index 00000000000..098e02595f5 --- /dev/null +++ b/src/Disks/ObjectStorages/S3/copyS3FileToDisk.cpp @@ -0,0 +1,69 @@ +#include + +#if USE_AWS_S3 + +#include +#include +#include +#include +#include + + +namespace DB +{ + +void copyS3FileToDisk( + const std::shared_ptr & s3_client, + const String & src_bucket, + const String & src_key, + const std::optional & version_id, + std::optional src_offset, + std::optional src_size, + DiskPtr destination_disk, + const String & destination_path, + WriteMode write_mode, + const ReadSettings & read_settings, + const WriteSettings & write_settings, + const S3Settings::RequestSettings & request_settings, + ThreadPoolCallbackRunner scheduler) +{ + if (!src_offset) + src_offset = 0; + + if (!src_size) + src_size = S3::getObjectSize(*s3_client, src_bucket, src_key, version_id.value_or(""), request_settings) - *src_offset; + + auto destination_data_source_description = destination_disk->getDataSourceDescription(); + if (destination_data_source_description != DataSourceDescription{DataSourceType::S3, s3_client->getInitialEndpoint(), false, false}) + { + LOG_TRACE(&Poco::Logger::get("copyS3FileToDisk"), "Copying {} to disk {} through buffers", src_key, destination_disk->getName()); + ReadBufferFromS3 read_buffer{s3_client, src_bucket, src_key, {}, request_settings, read_settings}; + if (*src_offset) + read_buffer.seek(*src_offset, SEEK_SET); + auto write_buffer = destination_disk->writeFile(destination_path, std::min(*src_size, DBMS_DEFAULT_BUFFER_SIZE), write_mode, write_settings); + copyData(read_buffer, *write_buffer, *src_size); + write_buffer->finalize(); + return; + } + + LOG_TRACE(&Poco::Logger::get("copyS3FileToDisk"), "Copying {} to disk {} using native copy", src_key, destination_disk->getName()); + + String dest_bucket = destination_disk->getObjectStorage()->getObjectsNamespace(); + + auto custom_write_object = [&](const StoredObject & object_, WriteMode write_mode_, const std::optional & object_attributes_) -> size_t + { + /// Object storage always uses mode `Rewrite` because it simulates append using metadata and different files. + chassert(write_mode_ == WriteMode::Rewrite); + + copyS3File(s3_client, src_bucket, src_key, *src_offset, *src_size, dest_bucket, /* dest_key= */ object_.absolute_path, + request_settings, object_attributes_, scheduler, /* for_disk_s3= */ true); + + return *src_size; + }; + + destination_disk->writeFileUsingCustomWriteObject(destination_path, write_mode, custom_write_object); +} + +} + +#endif diff --git a/src/Disks/ObjectStorages/S3/copyS3FileToDisk.h b/src/Disks/ObjectStorages/S3/copyS3FileToDisk.h new file mode 100644 index 00000000000..21c92ec9623 --- /dev/null +++ b/src/Disks/ObjectStorages/S3/copyS3FileToDisk.h @@ -0,0 +1,36 @@ +#pragma once + +#include "config.h" + +#if USE_AWS_S3 + +#include +#include +#include + + +namespace DB +{ + +/// Copies an object from S3 bucket to a disk of any type. +/// Depending on the disk the function can either do copying though buffers +/// (i.e. download the object by portions and then write those portions to the specified disk), +/// or perform a server-side copy. +void copyS3FileToDisk( + const std::shared_ptr & s3_client, + const String & src_bucket, + const String & src_key, + const std::optional & version_id, + std::optional src_offset, + std::optional src_size, + DiskPtr destination_disk, + const String & destination_path, + WriteMode write_mode = WriteMode::Rewrite, + const ReadSettings & read_settings = {}, + const WriteSettings & write_settings = {}, + const S3Settings::RequestSettings & request_settings = {}, + ThreadPoolCallbackRunner scheduler = {}); + +} + +#endif diff --git a/src/Disks/getDiskConfigurationFromAST.cpp b/src/Disks/getDiskConfigurationFromAST.cpp index e6b08046036..4b1323b4db8 100644 --- a/src/Disks/getDiskConfigurationFromAST.cpp +++ b/src/Disks/getDiskConfigurationFromAST.cpp @@ -83,4 +83,24 @@ DiskConfigurationPtr getDiskConfigurationFromAST(const std::string & root_name, return conf; } + +ASTs convertDiskConfigurationToAST(const Poco::Util::AbstractConfiguration & configuration, const std::string & config_path) +{ + ASTs result; + + Poco::Util::AbstractConfiguration::Keys keys; + configuration.keys(config_path, keys); + + for (const auto & key : keys) + { + result.push_back( + makeASTFunction( + "equals", + std::make_shared(key), + std::make_shared(configuration.getString(config_path + "." + key)))); + } + + return result; +} + } diff --git a/src/Disks/getDiskConfigurationFromAST.h b/src/Disks/getDiskConfigurationFromAST.h index 1f9d7c1bfe6..5697955e914 100644 --- a/src/Disks/getDiskConfigurationFromAST.h +++ b/src/Disks/getDiskConfigurationFromAST.h @@ -25,4 +25,12 @@ using DiskConfigurationPtr = Poco::AutoPtr; */ DiskConfigurationPtr getDiskConfigurationFromAST(const std::string & root_name, const ASTs & disk_args, ContextPtr context); +/// The same as above function, but return XML::Document for easier modification of result configuration. +[[ maybe_unused ]] Poco::AutoPtr getDiskConfigurationFromASTImpl(const std::string & root_name, const ASTs & disk_args, ContextPtr context); + +/* + * A reverse function. + */ +[[ maybe_unused ]] ASTs convertDiskConfigurationToAST(const Poco::Util::AbstractConfiguration & configuration, const std::string & config_path); + } diff --git a/src/Disks/getOrCreateDiskFromAST.cpp b/src/Disks/getOrCreateDiskFromAST.cpp index 5deb9ab11b5..637acff7b95 100644 --- a/src/Disks/getOrCreateDiskFromAST.cpp +++ b/src/Disks/getOrCreateDiskFromAST.cpp @@ -6,9 +6,13 @@ #include #include #include +#include +#include #include #include #include +#include +#include namespace DB { @@ -18,43 +22,85 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; } -std::string getOrCreateDiskFromDiskAST(const ASTFunction & function, ContextPtr context) +namespace { - /// We need a unique name for a created custom disk, but it needs to be the same - /// after table is reattached or server is restarted, so take a hash of the disk - /// configuration serialized ast as a disk name suffix. - auto disk_setting_string = serializeAST(function, true); - auto disk_name = DiskSelector::TMP_INTERNAL_DISK_PREFIX - + toString(sipHash128(disk_setting_string.data(), disk_setting_string.size())); - - auto result_disk = context->getOrCreateDisk(disk_name, [&](const DisksMap & disks_map) -> DiskPtr { - const auto * function_args_expr = assert_cast(function.arguments.get()); - const auto & function_args = function_args_expr->children; - auto config = getDiskConfigurationFromAST(disk_name, function_args, context); - auto disk = DiskFactory::instance().create(disk_name, *config, disk_name, context, disks_map); - /// Mark that disk can be used without storage policy. - disk->markDiskAsCustom(); - return disk; - }); - - if (!result_disk->isRemote()) + std::string getOrCreateDiskFromDiskAST(const ASTFunction & function, ContextPtr context) { - static constexpr auto custom_disks_base_dir_in_config = "custom_local_disks_base_directory"; - auto disk_path_expected_prefix = context->getConfigRef().getString(custom_disks_base_dir_in_config, ""); + /// We need a unique name for a created custom disk, but it needs to be the same + /// after table is reattached or server is restarted, so take a hash of the disk + /// configuration serialized ast as a disk name suffix. + auto disk_setting_string = serializeAST(function, true); + auto disk_name = DiskSelector::TMP_INTERNAL_DISK_PREFIX + + toString(sipHash128(disk_setting_string.data(), disk_setting_string.size())); - if (disk_path_expected_prefix.empty()) - throw Exception( - ErrorCodes::BAD_ARGUMENTS, - "Base path for custom local disks must be defined in config file by `{}`", - custom_disks_base_dir_in_config); + auto result_disk = context->getOrCreateDisk(disk_name, [&](const DisksMap & disks_map) -> DiskPtr { + const auto * function_args_expr = assert_cast(function.arguments.get()); + const auto & function_args = function_args_expr->children; + auto config = getDiskConfigurationFromAST(disk_name, function_args, context); + auto disk = DiskFactory::instance().create(disk_name, *config, disk_name, context, disks_map); + /// Mark that disk can be used without storage policy. + disk->markDiskAsCustom(); + return disk; + }); - if (!pathStartsWith(result_disk->getPath(), disk_path_expected_prefix)) - throw Exception( - ErrorCodes::BAD_ARGUMENTS, - "Path of the custom local disk must be inside `{}` directory", - disk_path_expected_prefix); + if (!result_disk->isRemote()) + { + static constexpr auto custom_disks_base_dir_in_config = "custom_local_disks_base_directory"; + auto disk_path_expected_prefix = context->getConfigRef().getString(custom_disks_base_dir_in_config, ""); + + if (disk_path_expected_prefix.empty()) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Base path for custom local disks must be defined in config file by `{}`", + custom_disks_base_dir_in_config); + + if (!pathStartsWith(result_disk->getPath(), disk_path_expected_prefix)) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Path of the custom local disk must be inside `{}` directory", + disk_path_expected_prefix); + } + + return disk_name; } + class DiskConfigurationFlattener + { + public: + struct Data + { + ContextPtr context; + }; + + static bool needChildVisit(const ASTPtr &, const ASTPtr &) { return true; } + + static void visit(ASTPtr & ast, Data & data) + { + if (isDiskFunction(ast)) + { + auto disk_name = getOrCreateDiskFromDiskAST(*ast->as(), data.context); + ast = std::make_shared(disk_name); + } + } + }; + + /// Visits children first. + using FlattenDiskConfigurationVisitor = InDepthNodeVisitor; +} + + +std::string getOrCreateDiskFromDiskAST(const ASTPtr & disk_function, ContextPtr context) +{ + if (!isDiskFunction(disk_function)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Expected a disk function"); + + auto ast = disk_function->clone(); + + FlattenDiskConfigurationVisitor::Data data{context}; + FlattenDiskConfigurationVisitor{data}.visit(ast); + + auto disk_name = assert_cast(*ast).value.get(); + LOG_TRACE(&Poco::Logger::get("getOrCreateDiskFromDiskAST"), "Result disk name: {}", disk_name); return disk_name; } diff --git a/src/Disks/getOrCreateDiskFromAST.h b/src/Disks/getOrCreateDiskFromAST.h index 7c64707b0bd..0195f575278 100644 --- a/src/Disks/getOrCreateDiskFromAST.h +++ b/src/Disks/getOrCreateDiskFromAST.h @@ -13,6 +13,6 @@ class ASTFunction; * add it to DiskSelector by a unique (but always the same for given configuration) disk name * and return this name. */ -std::string getOrCreateDiskFromDiskAST(const ASTFunction & function, ContextPtr context); +std::string getOrCreateDiskFromDiskAST(const ASTPtr & disk_function, ContextPtr context); } diff --git a/src/Formats/BSONTypes.cpp b/src/Formats/BSONTypes.cpp index 813c155325a..88396fd2aba 100644 --- a/src/Formats/BSONTypes.cpp +++ b/src/Formats/BSONTypes.cpp @@ -1,6 +1,6 @@ #include #include -#include +#include namespace DB { diff --git a/src/Formats/CapnProtoUtils.cpp b/src/Formats/CapnProtoUtils.cpp index e5f619faff5..a557b762672 100644 --- a/src/Formats/CapnProtoUtils.cpp +++ b/src/Formats/CapnProtoUtils.cpp @@ -317,6 +317,7 @@ static bool checkCapnProtoType(const capnp::Type & capnp_type, const DataTypePtr case TypeIndex::UInt16: return capnp_type.isUInt16(); case TypeIndex::DateTime: [[fallthrough]]; + case TypeIndex::IPv4: [[fallthrough]]; case TypeIndex::UInt32: return capnp_type.isUInt32(); case TypeIndex::UInt64: @@ -355,6 +356,7 @@ static bool checkCapnProtoType(const capnp::Type & capnp_type, const DataTypePtr case TypeIndex::LowCardinality: return checkCapnProtoType(capnp_type, assert_cast(data_type.get())->getDictionaryType(), mode, error_message, column_name); case TypeIndex::FixedString: [[fallthrough]]; + case TypeIndex::IPv6: [[fallthrough]]; case TypeIndex::String: return capnp_type.isText() || capnp_type.isData(); default: diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp index aca3166a8c4..a951a7fdd92 100644 --- a/src/Formats/FormatFactory.cpp +++ b/src/Formats/FormatFactory.cpp @@ -118,6 +118,7 @@ FormatSettings getFormatSettings(ContextPtr context, const Settings & settings) format_settings.parquet.output_string_as_string = settings.output_format_parquet_string_as_string; format_settings.parquet.output_fixed_string_as_fixed_byte_array = settings.output_format_parquet_fixed_string_as_fixed_byte_array; format_settings.parquet.max_block_size = settings.input_format_parquet_max_block_size; + format_settings.parquet.output_compression_method = settings.output_format_parquet_compression_method; format_settings.pretty.charset = settings.output_format_pretty_grid_charset.toString() == "ASCII" ? FormatSettings::Pretty::Charset::ASCII : FormatSettings::Pretty::Charset::UTF8; format_settings.pretty.color = settings.output_format_pretty_color; format_settings.pretty.max_column_pad_width = settings.output_format_pretty_max_column_pad_width; @@ -158,6 +159,7 @@ FormatSettings getFormatSettings(ContextPtr context, const Settings & settings) format_settings.arrow.case_insensitive_column_matching = settings.input_format_arrow_case_insensitive_column_matching; format_settings.arrow.output_string_as_string = settings.output_format_arrow_string_as_string; format_settings.arrow.output_fixed_string_as_fixed_byte_array = settings.output_format_arrow_fixed_string_as_fixed_byte_array; + format_settings.arrow.output_compression_method = settings.output_format_arrow_compression_method; format_settings.orc.import_nested = settings.input_format_orc_import_nested; format_settings.orc.allow_missing_columns = settings.input_format_orc_allow_missing_columns; format_settings.orc.row_batch_size = settings.input_format_orc_row_batch_size; @@ -168,6 +170,7 @@ FormatSettings getFormatSettings(ContextPtr context, const Settings & settings) format_settings.orc.skip_columns_with_unsupported_types_in_schema_inference = settings.input_format_orc_skip_columns_with_unsupported_types_in_schema_inference; format_settings.orc.case_insensitive_column_matching = settings.input_format_orc_case_insensitive_column_matching; format_settings.orc.output_string_as_string = settings.output_format_orc_string_as_string; + format_settings.orc.output_compression_method = settings.output_format_orc_compression_method; format_settings.defaults_for_omitted_fields = settings.input_format_defaults_for_omitted_fields; format_settings.capn_proto.enum_comparing_mode = settings.format_capn_proto_enum_comparising_mode; format_settings.capn_proto.skip_fields_with_unsupported_types_in_schema_inference = settings.input_format_capn_proto_skip_fields_with_unsupported_types_in_schema_inference; @@ -191,6 +194,7 @@ FormatSettings getFormatSettings(ContextPtr context, const Settings & settings) format_settings.bson.output_string_as_string = settings.output_format_bson_string_as_string; format_settings.bson.skip_fields_with_unsupported_types_in_schema_inference = settings.input_format_bson_skip_fields_with_unsupported_types_in_schema_inference; format_settings.max_binary_string_size = settings.format_binary_max_string_size; + format_settings.native.allow_types_conversion = settings.input_format_native_allow_types_conversion; format_settings.max_parser_depth = context->getSettingsRef().max_parser_depth; format_settings.client_protocol_version = context->getClientProtocolVersion(); diff --git a/src/Formats/FormatSettings.h b/src/Formats/FormatSettings.h index d1755a35c5f..7be7b5b98aa 100644 --- a/src/Formats/FormatSettings.h +++ b/src/Formats/FormatSettings.h @@ -86,6 +86,13 @@ struct FormatSettings UInt64 max_parser_depth = DBMS_DEFAULT_MAX_PARSER_DEPTH; + enum class ArrowCompression + { + NONE, + LZ4_FRAME, + ZSTD + }; + struct { UInt64 row_group_size = 1000000; @@ -96,6 +103,7 @@ struct FormatSettings bool case_insensitive_column_matching = false; bool output_string_as_string = false; bool output_fixed_string_as_fixed_byte_array = true; + ArrowCompression output_compression_method = ArrowCompression::NONE; } arrow; struct @@ -183,6 +191,16 @@ struct FormatSettings V2_LATEST, }; + enum class ParquetCompression + { + NONE, + SNAPPY, + ZSTD, + LZ4, + GZIP, + BROTLI, + }; + struct { UInt64 row_group_size = 1000000; @@ -195,6 +213,7 @@ struct FormatSettings bool output_fixed_string_as_fixed_byte_array = true; UInt64 max_block_size = 8192; ParquetVersion output_version; + ParquetCompression output_compression_method = ParquetCompression::SNAPPY; } parquet; struct Pretty @@ -276,6 +295,15 @@ struct FormatSettings bool accurate_types_of_literals = true; } values; + enum class ORCCompression + { + NONE, + LZ4, + SNAPPY, + ZSTD, + ZLIB, + }; + struct { bool import_nested = false; @@ -285,6 +313,7 @@ struct FormatSettings bool case_insensitive_column_matching = false; std::unordered_set skip_stripes = {}; bool output_string_as_string = false; + ORCCompression output_compression_method = ORCCompression::NONE; } orc; /// For capnProto format we should determine how to @@ -335,6 +364,11 @@ struct FormatSettings bool output_string_as_string; bool skip_fields_with_unsupported_types_in_schema_inference; } bson; + + struct + { + bool allow_types_conversion = true; + } native; }; } diff --git a/src/Formats/MarkInCompressedFile.cpp b/src/Formats/MarkInCompressedFile.cpp new file mode 100644 index 00000000000..41f6152dc13 --- /dev/null +++ b/src/Formats/MarkInCompressedFile.cpp @@ -0,0 +1,103 @@ +#include + +#include + +namespace DB +{ + +// Write a range of bits in a bit-packed array. +// The array must be overallocated by one element. +// The bit range must be pre-filled with zeros. +void writeBits(UInt64 * dest, size_t bit_offset, UInt64 value) +{ + size_t mod = bit_offset % 64; + dest[bit_offset / 64] |= value << mod; + if (mod) + dest[bit_offset / 64 + 1] |= value >> (64 - mod); +} + +// The array must be overallocated by one element. +UInt64 readBits(const UInt64 * src, size_t bit_offset, size_t num_bits) +{ + size_t mod = bit_offset % 64; + UInt64 value = src[bit_offset / 64] >> mod; + if (mod) + value |= src[bit_offset / 64 + 1] << (64 - mod); + return value & maskLowBits(num_bits); +} + +MarksInCompressedFile::MarksInCompressedFile(const PlainArray & marks) + : num_marks(marks.size()), blocks((marks.size() + MARKS_PER_BLOCK - 1) / MARKS_PER_BLOCK, BlockInfo{}) +{ + if (num_marks == 0) + { + return; + } + + // First pass: calculate layout of all blocks and total memory required. + size_t packed_bits = 0; + for (size_t block_idx = 0; block_idx < blocks.size(); ++block_idx) + { + BlockInfo & block = blocks[block_idx]; + block.bit_offset_in_packed_array = packed_bits; + + size_t max_x = 0; + size_t max_y = 0; + size_t num_marks_in_this_block = std::min(MARKS_PER_BLOCK, num_marks - block_idx * MARKS_PER_BLOCK); + for (size_t i = 0; i < num_marks_in_this_block; ++i) + { + const auto & mark = marks[block_idx * MARKS_PER_BLOCK + i]; + block.min_x = std::min(block.min_x, mark.offset_in_compressed_file); + max_x = std::max(max_x, mark.offset_in_compressed_file); + block.min_y = std::min(block.min_y, mark.offset_in_decompressed_block); + max_y = std::max(max_y, mark.offset_in_decompressed_block); + + block.trailing_zero_bits_in_y + = std::min(block.trailing_zero_bits_in_y, static_cast(getTrailingZeroBits(mark.offset_in_decompressed_block))); + } + + block.bits_for_x = sizeof(size_t) * 8 - getLeadingZeroBits(max_x - block.min_x); + block.bits_for_y = sizeof(size_t) * 8 - getLeadingZeroBits((max_y - block.min_y) >> block.trailing_zero_bits_in_y); + packed_bits += num_marks_in_this_block * (block.bits_for_x + block.bits_for_y); + } + + // Overallocate by +1 element to let the bit packing/unpacking do less bounds checking. + size_t packed_length = (packed_bits + 63) / 64 + 1; + packed.reserve_exact(packed_length); + packed.resize_fill(packed_length); + + // Second pass: write out the packed marks. + for (size_t idx = 0; idx < num_marks; ++idx) + { + const auto & mark = marks[idx]; + auto [block, offset] = lookUpMark(idx); + writeBits(packed.data(), offset, mark.offset_in_compressed_file - block->min_x); + writeBits( + packed.data(), + offset + block->bits_for_x, + (mark.offset_in_decompressed_block - block->min_y) >> block->trailing_zero_bits_in_y); + } +} + +MarkInCompressedFile MarksInCompressedFile::get(size_t idx) const +{ + auto [block, offset] = lookUpMark(idx); + size_t x = block->min_x + readBits(packed.data(), offset, block->bits_for_x); + size_t y = block->min_y + (readBits(packed.data(), offset + block->bits_for_x, block->bits_for_y) << block->trailing_zero_bits_in_y); + return MarkInCompressedFile{.offset_in_compressed_file = x, .offset_in_decompressed_block = y}; +} + +std::tuple MarksInCompressedFile::lookUpMark(size_t idx) const +{ + size_t block_idx = idx / MARKS_PER_BLOCK; + const BlockInfo & block = blocks[block_idx]; + size_t offset = block.bit_offset_in_packed_array + (idx - block_idx * MARKS_PER_BLOCK) * (block.bits_for_x + block.bits_for_y); + return {&block, offset}; +} + +size_t MarksInCompressedFile::approximateMemoryUsage() const +{ + return sizeof(*this) + blocks.size() * sizeof(blocks[0]) + packed.size() * sizeof(packed[0]); +} + +} diff --git a/src/Formats/MarkInCompressedFile.h b/src/Formats/MarkInCompressedFile.h index 287d3f7909d..08e4f182c45 100644 --- a/src/Formats/MarkInCompressedFile.h +++ b/src/Formats/MarkInCompressedFile.h @@ -2,8 +2,8 @@ #include -#include #include +#include #include @@ -23,15 +23,9 @@ struct MarkInCompressedFile return std::tie(offset_in_compressed_file, offset_in_decompressed_block) == std::tie(rhs.offset_in_compressed_file, rhs.offset_in_decompressed_block); } - bool operator!=(const MarkInCompressedFile & rhs) const - { - return !(*this == rhs); - } + bool operator!=(const MarkInCompressedFile & rhs) const { return !(*this == rhs); } - auto asTuple() const - { - return std::make_tuple(offset_in_compressed_file, offset_in_decompressed_block); - } + auto asTuple() const { return std::make_tuple(offset_in_compressed_file, offset_in_decompressed_block); } String toString() const { @@ -40,20 +34,87 @@ struct MarkInCompressedFile String toStringWithRows(size_t rows_num) const { - return "(" + DB::toString(offset_in_compressed_file) + "," + DB::toString(offset_in_decompressed_block) + "," + DB::toString(rows_num) + ")"; + return "(" + DB::toString(offset_in_compressed_file) + "," + DB::toString(offset_in_decompressed_block) + "," + + DB::toString(rows_num) + ")"; } - }; -class MarksInCompressedFile : public PODArray +/** + * In-memory representation of an array of marks. + * + * Uses an ad-hoc compression scheme that decreases memory usage while allowing + * random access in O(1) time. + * This is independent from the marks *file* format, which may be uncompressed + * or use a different compression method. + * + * Typical memory usage: + * * ~3 bytes/mark for integer columns + * * ~5 bytes/mark for string columns + * * ~0.3 bytes/mark for trivial marks in auxiliary dict files of LowCardinality columns + */ +class MarksInCompressedFile { public: - explicit MarksInCompressedFile(size_t n) : PODArray(n) {} + using PlainArray = PODArray; - void read(ReadBuffer & buffer, size_t from, size_t count) + MarksInCompressedFile(const PlainArray & marks); + + MarkInCompressedFile get(size_t idx) const; + + size_t approximateMemoryUsage() const; + +private: + /** Throughout this class: + * * "x" stands for offset_in_compressed_file, + * * "y" stands for offset_in_decompressed_block. + */ + + /** We need to store a sequence of marks, each consisting of two 64-bit integers: + * offset_in_compressed_file and offset_in_decompressed_block. We'll call them x and y for + * convenience, since compression doesn't care what they mean. The compression exploits the + * following regularities: + * * y is usually zero. + * * x usually increases steadily. + * * Differences between x values in nearby marks usually fit in much fewer than 64 bits. + * + * We split the sequence of marks into blocks, each containing MARKS_PER_BLOCK marks. + * (Not to be confused with data blocks.) + * For each mark, we store the difference [value] - [min value in the block], for each of the + * two values in the mark. Each block specifies the number of bits to use for these differences + * for all marks in this block. + * The smaller the blocks the fewer bits are required, but the bigger the relative overhead of + * block headers. + * + * Packed marks and block headers all live in one contiguous array. + */ + + struct BlockInfo { - buffer.readStrict(reinterpret_cast(data() + from), count * sizeof(MarkInCompressedFile)); - } + // Min offset_in_compressed_file and offset_in_decompressed_block, correspondingly. + size_t min_x = UINT64_MAX; + size_t min_y = UINT64_MAX; + + // Place in `packed` where this block start. + size_t bit_offset_in_packed_array; + + // How many bits each mark takes. These numbers are bit-packed in the `packed` array. + // Can be zero. (Especially for y, which is typically all zeroes.) + UInt8 bits_for_x; + UInt8 bits_for_y; + // The `y` values should be <<'ed by this amount. + // Useful for integer columns when marks granularity is a power of 2; in this case all + // offset_in_decompressed_block values are divisible by 2^15 or so. + UInt8 trailing_zero_bits_in_y = 63; + }; + + static constexpr size_t MARKS_PER_BLOCK = 256; + + size_t num_marks; + PODArray blocks; + PODArray packed; + + // Mark idx -> {block info, bit offset in `packed`}. + std::tuple lookUpMark(size_t idx) const; }; } diff --git a/src/Formats/NativeReader.cpp b/src/Formats/NativeReader.cpp index 9f8d4ba1930..eca88a41c13 100644 --- a/src/Formats/NativeReader.cpp +++ b/src/Formats/NativeReader.cpp @@ -15,6 +15,8 @@ #include #include +#include + namespace DB { @@ -39,12 +41,14 @@ NativeReader::NativeReader( UInt64 server_revision_, bool skip_unknown_columns_, bool null_as_default_, + bool allow_types_conversion_, BlockMissingValues * block_missing_values_) : istr(istr_) , header(header_) , server_revision(server_revision_) , skip_unknown_columns(skip_unknown_columns_) , null_as_default(null_as_default_) + , allow_types_conversion(allow_types_conversion_) , block_missing_values(block_missing_values_) { } @@ -204,11 +208,31 @@ Block NativeReader::read() if (null_as_default) insertNullAsDefaultIfNeeded(column, header_column, header.getPositionByName(column.name), block_missing_values); - /// Support insert from old clients without low cardinality type. if (!header_column.type->equals(*column.type)) { - column.column = recursiveTypeConversion(column.column, column.type, header.safeGetByPosition(i).type); - column.type = header.safeGetByPosition(i).type; + if (allow_types_conversion) + { + try + { + column.column = castColumn(column, header_column.type); + } + catch (Exception & e) + { + e.addMessage(fmt::format( + "while converting column \"{}\" from type {} to type {}", + column.name, + column.type->getName(), + header_column.type->getName())); + throw; + } + } + else + { + /// Support insert from old clients without low cardinality type. + column.column = recursiveLowCardinalityTypeConversion(column.column, column.type, header_column.type); + } + + column.type = header_column.type; } } else diff --git a/src/Formats/NativeReader.h b/src/Formats/NativeReader.h index 2d8b16e06eb..3cec4afd997 100644 --- a/src/Formats/NativeReader.h +++ b/src/Formats/NativeReader.h @@ -30,6 +30,7 @@ public: UInt64 server_revision_, bool skip_unknown_columns_ = false, bool null_as_default_ = false, + bool allow_types_conversion_ = false, BlockMissingValues * block_missing_values_ = nullptr); /// For cases when we have an index. It allows to skip columns. Only columns specified in the index will be read. @@ -51,6 +52,7 @@ private: UInt64 server_revision; bool skip_unknown_columns = false; bool null_as_default = false; + bool allow_types_conversion = false; BlockMissingValues * block_missing_values = nullptr; bool use_index = false; diff --git a/src/Formats/SchemaInferenceUtils.cpp b/src/Formats/SchemaInferenceUtils.cpp index 7a242a9f81c..011860948c3 100644 --- a/src/Formats/SchemaInferenceUtils.cpp +++ b/src/Formats/SchemaInferenceUtils.cpp @@ -131,6 +131,7 @@ namespace type_indexes.erase(TypeIndex::Date); type_indexes.erase(TypeIndex::DateTime); + type_indexes.insert(TypeIndex::String); return; } @@ -983,13 +984,16 @@ DataTypePtr tryInferNumberFromString(std::string_view field, const FormatSetting if (tryReadIntText(tmp_int, buf) && buf.eof()) return std::make_shared(); + /// We can safely get back to the start of buffer, because we read from a string and we didn't reach eof. + buf.position() = buf.buffer().begin(); + /// In case of Int64 overflow, try to infer UInt64 UInt64 tmp_uint; if (tryReadIntText(tmp_uint, buf) && buf.eof()) return std::make_shared(); } - /// We cam safely get back to the start of buffer, because we read from a string and we didn't reach eof. + /// We can safely get back to the start of buffer, because we read from a string and we didn't reach eof. buf.position() = buf.buffer().begin(); Float64 tmp; diff --git a/src/Formats/tests/gtest_marks.cpp b/src/Formats/tests/gtest_marks.cpp new file mode 100644 index 00000000000..334e99d2ec6 --- /dev/null +++ b/src/Formats/tests/gtest_marks.cpp @@ -0,0 +1,52 @@ +#include +#include + +#include + +using namespace DB; + +TEST(Marks, Compression) +{ + std::random_device dev; + std::mt19937 rng(dev()); + + auto gen = [&](size_t count, size_t max_x_increment, size_t max_y_increment) + { + size_t x = 0, y = 0; + PODArray plain(count); + for (int i = 0; i < count; ++i) + { + x += rng() % (max_x_increment + 1); + y += rng() % (max_y_increment + 1); + plain[i] = MarkInCompressedFile{.offset_in_compressed_file = x, .offset_in_decompressed_block = y}; + } + return plain; + }; + + auto test = [](const PODArray & plain, size_t max_bits_per_mark) + { + PODArray copy; + copy.assign(plain); // paranoid in case next line mutates it + + MarksInCompressedFile marks(copy); + for (size_t i = 0; i < plain.size(); ++i) + ASSERT_EQ(marks.get(i), plain[i]); + + EXPECT_LE((marks.approximateMemoryUsage() - sizeof(MarksInCompressedFile)) * 8, plain.size() * max_bits_per_mark); + }; + + // Typical. + test(gen(10000, 1'000'000, 0), 30); + + // Completely random 64-bit values. + test(gen(10000, UINT64_MAX - 1, UINT64_MAX - 1), 130); + + // All zeros. + test(gen(10000, 0, 0), 2); + + // Short. + test(gen(10, 1000, 1000), 65); + + // Empty. + test(gen(0, 0, 0), 0); +} diff --git a/src/Formats/verbosePrintString.cpp b/src/Formats/verbosePrintString.cpp index 2f3e09ed75f..5c6111c2929 100644 --- a/src/Formats/verbosePrintString.cpp +++ b/src/Formats/verbosePrintString.cpp @@ -1,5 +1,5 @@ #include -#include +#include #include diff --git a/src/Functions/DateTimeTransforms.h b/src/Functions/DateTimeTransforms.h index b1d2c1e2ecf..f179d9fbe60 100644 --- a/src/Functions/DateTimeTransforms.h +++ b/src/Functions/DateTimeTransforms.h @@ -53,6 +53,10 @@ struct ToDateImpl { static constexpr auto name = "toDate"; + static inline UInt16 execute(const DecimalUtils::DecimalComponents & t, const DateLUTImpl & time_zone) + { + return static_cast(time_zone.toDayNum(t.whole)); + } static inline UInt16 execute(Int64 t, const DateLUTImpl & time_zone) { return UInt16(time_zone.toDayNum(t)); @@ -69,6 +73,10 @@ struct ToDateImpl { return d; } + static inline DecimalUtils::DecimalComponents executeExtendedResult(const DecimalUtils::DecimalComponents & t, const DateLUTImpl & time_zone) + { + return {time_zone.toDayNum(t.whole), 0}; + } using FactorTransform = ZeroTransform; }; diff --git a/src/Functions/FunctionFile.cpp b/src/Functions/FunctionFile.cpp index 240732965f4..fa7dda82e1c 100644 --- a/src/Functions/FunctionFile.cpp +++ b/src/Functions/FunctionFile.cpp @@ -38,6 +38,8 @@ public: String getName() const override { return name; } size_t getNumberOfArguments() const override { return 0; } bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; } + bool isDeterministic() const override { return false; } + bool isDeterministicInScopeOfQuery() const override { return false; } DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override { diff --git a/src/Functions/FunctionsCodingIP.cpp b/src/Functions/FunctionsCodingIP.cpp index a941092b7d6..4784368db9b 100644 --- a/src/Functions/FunctionsCodingIP.cpp +++ b/src/Functions/FunctionsCodingIP.cpp @@ -26,7 +26,7 @@ #include #include #include -#include +#include #include #include diff --git a/src/Functions/FunctionsCodingUUID.cpp b/src/Functions/FunctionsCodingUUID.cpp index dade406c801..dd9170e44ad 100644 --- a/src/Functions/FunctionsCodingUUID.cpp +++ b/src/Functions/FunctionsCodingUUID.cpp @@ -3,7 +3,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/src/Functions/FunctionsEmbeddedDictionaries.h b/src/Functions/FunctionsEmbeddedDictionaries.h index e54ab0277d5..af040c6ab93 100644 --- a/src/Functions/FunctionsEmbeddedDictionaries.h +++ b/src/Functions/FunctionsEmbeddedDictionaries.h @@ -166,12 +166,12 @@ public: if (arguments[0]->getName() != TypeName) throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of argument of function {} (must be {})", - arguments[0]->getName(), getName(), String(TypeName)); + arguments[0]->getName(), getName(), TypeName); - if (arguments.size() == 2 && arguments[1]->getName() != TypeName) + if (arguments.size() == 2 && arguments[1]->getName() != "String") throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, - "Illegal type {} of the second ('point of view') argument of function {} (must be {})", - arguments[1]->getName(), getName(), String(TypeName)); + "Illegal type {} of the second ('point of view') argument of function {} (must be String)", + arguments[1]->getName(), getName()); return arguments[0]; } @@ -257,16 +257,16 @@ public: if (arguments[0]->getName() != TypeName) throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of first argument of function {} (must be {})", - arguments[0]->getName(), getName(), String(TypeName)); + arguments[0]->getName(), getName(), TypeName); if (arguments[1]->getName() != TypeName) throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of second argument of function {} (must be {})", - arguments[1]->getName(), getName(), String(TypeName)); + arguments[1]->getName(), getName(), TypeName); - if (arguments.size() == 3 && arguments[2]->getName() != TypeName) + if (arguments.size() == 3 && arguments[2]->getName() != "String") throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, - "Illegal type {} of the third ('point of view') argument of function {} (must be {})", - arguments[2]->getName(), getName(), String(TypeName)); + "Illegal type {} of the third ('point of view') argument of function {} (must be String)", + arguments[2]->getName(), getName()); return std::make_shared(); } @@ -390,12 +390,12 @@ public: if (arguments[0]->getName() != TypeName) throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of argument of function {} (must be {})", - arguments[0]->getName(), getName(), String(TypeName)); + arguments[0]->getName(), getName(), TypeName); - if (arguments.size() == 2 && arguments[1]->getName() != TypeName) + if (arguments.size() == 2 && arguments[1]->getName() != "String") throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, - "Illegal type {} of the second ('point of view') argument of function {} (must be {})", - arguments[1]->getName(), getName(), String(TypeName)); + "Illegal type {} of the second ('point of view') argument of function {} (must be String)", + arguments[1]->getName(), getName()); return std::make_shared(arguments[0]); } @@ -591,15 +591,15 @@ public: "Number of arguments for function {} doesn't match: passed {}, should be 1 or 2.", getName(), arguments.size()); - if (arguments[0]->getName() != TypeName) + if (arguments[0]->getName() != "UInt32") throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, - "Illegal type {} of the first argument of function {} (must be {})", - arguments[0]->getName(), getName(), String(TypeName)); + "Illegal type {} of the first argument of function {} (must be UInt32)", + arguments[0]->getName(), getName()); - if (arguments.size() == 2 && arguments[1]->getName() != TypeName) + if (arguments.size() == 2 && arguments[1]->getName() != "String") throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, - "Illegal type {} of the second argument of function {} (must be {})", - arguments[0]->getName(), getName(), String(TypeName)); + "Illegal type {} of the second argument of function {} (must be String)", + arguments[0]->getName(), getName()); return std::make_shared(); } diff --git a/src/Functions/FunctionsStringSimilarity.cpp b/src/Functions/FunctionsStringSimilarity.cpp index 0cc0248baf4..faa01abb675 100644 --- a/src/Functions/FunctionsStringSimilarity.cpp +++ b/src/Functions/FunctionsStringSimilarity.cpp @@ -285,9 +285,9 @@ struct NgramDistanceImpl size_t first_size = dispatchSearcher(calculateHaystackStatsAndMetric, data.data(), data_size, common_stats.get(), distance, nullptr); /// For !symmetric version we should not use first_size. if constexpr (symmetric) - res = distance * 1.f / std::max(first_size + second_size, static_cast(1)); + res = distance * 1.f / std::max(first_size + second_size, 1uz); else - res = 1.f - distance * 1.f / std::max(second_size, static_cast(1)); + res = 1.f - distance * 1.f / std::max(second_size, 1uz); } else { @@ -353,9 +353,9 @@ struct NgramDistanceImpl /// For !symmetric version we should not use haystack_stats_size. if constexpr (symmetric) - res[i] = distance * 1.f / std::max(haystack_stats_size + needle_stats_size, static_cast(1)); + res[i] = distance * 1.f / std::max(haystack_stats_size + needle_stats_size, 1uz); else - res[i] = 1.f - distance * 1.f / std::max(needle_stats_size, static_cast(1)); + res[i] = 1.f - distance * 1.f / std::max(needle_stats_size, 1uz); } else { @@ -424,7 +424,7 @@ struct NgramDistanceImpl for (size_t j = 0; j < needle_stats_size; ++j) --common_stats[needle_ngram_storage[j]]; - res[i] = 1.f - distance * 1.f / std::max(needle_stats_size, static_cast(1)); + res[i] = 1.f - distance * 1.f / std::max(needle_stats_size, 1uz); } else { @@ -471,9 +471,9 @@ struct NgramDistanceImpl ngram_storage.get()); /// For !symmetric version we should not use haystack_stats_size. if constexpr (symmetric) - res[i] = distance * 1.f / std::max(haystack_stats_size + needle_stats_size, static_cast(1)); + res[i] = distance * 1.f / std::max(haystack_stats_size + needle_stats_size, 1uz); else - res[i] = 1.f - distance * 1.f / std::max(needle_stats_size, static_cast(1)); + res[i] = 1.f - distance * 1.f / std::max(needle_stats_size, 1uz); } else { diff --git a/src/Functions/IFunction.cpp b/src/Functions/IFunction.cpp index 1f25794536b..4081850bb86 100644 --- a/src/Functions/IFunction.cpp +++ b/src/Functions/IFunction.cpp @@ -323,7 +323,7 @@ ColumnPtr IExecutableFunction::execute(const ColumnsWithTypeAndName & arguments, const auto * column_sparse = checkAndGetColumn(arguments[i].column.get()); /// In rare case, when sparse column doesn't have default values, /// it's more convenient to convert it to full before execution of function. - if (column_sparse && column_sparse->getNumberOfDefaults()) + if (column_sparse && column_sparse->getNumberOfDefaultRows()) { sparse_column_position = i; ++num_sparse_columns; @@ -361,7 +361,9 @@ ColumnPtr IExecutableFunction::execute(const ColumnsWithTypeAndName & arguments, return res->cloneResized(input_rows_count); /// If default of sparse column is changed after execution of function, convert to full column. - if (!result_type->supportsSparseSerialization() || !res->isDefaultAt(0)) + /// If there are any default in non-zero position after execution of function, convert to full column. + /// Currently there is no easy way to rebuild sparse column with new offsets. + if (!result_type->supportsSparseSerialization() || !res->isDefaultAt(0) || res->getNumberOfDefaultRows() != 1) { const auto & offsets_data = assert_cast &>(*sparse_offsets).getData(); return res->createWithOffsets(offsets_data, (*res)[0], input_rows_count, /*shift=*/ 1); diff --git a/src/Functions/IFunctionDateOrDateTime.h b/src/Functions/IFunctionDateOrDateTime.h index 343511f0f91..5d3f28bcce8 100644 --- a/src/Functions/IFunctionDateOrDateTime.h +++ b/src/Functions/IFunctionDateOrDateTime.h @@ -46,36 +46,57 @@ public: { if constexpr (std::is_same_v) return { .is_monotonic = true, .is_always_monotonic = true }; - - const IFunction::Monotonicity is_monotonic = { .is_monotonic = true }; - const IFunction::Monotonicity is_not_monotonic; - - const DateLUTImpl * date_lut = &DateLUT::instance(); - if (const auto * timezone = dynamic_cast(&type)) - date_lut = &timezone->getTimeZone(); - - if (left.isNull() || right.isNull()) - return is_not_monotonic; - - /// The function is monotonous on the [left, right] segment, if the factor transformation returns the same values for them. - - if (checkAndGetDataType(&type)) - { - return Transform::FactorTransform::execute(UInt16(left.get()), *date_lut) - == Transform::FactorTransform::execute(UInt16(right.get()), *date_lut) - ? is_monotonic : is_not_monotonic; - } - else if (checkAndGetDataType(&type)) - { - return Transform::FactorTransform::execute(Int32(left.get()), *date_lut) - == Transform::FactorTransform::execute(Int32(right.get()), *date_lut) - ? is_monotonic : is_not_monotonic; - } else { - return Transform::FactorTransform::execute(UInt32(left.get()), *date_lut) - == Transform::FactorTransform::execute(UInt32(right.get()), *date_lut) - ? is_monotonic : is_not_monotonic; + const IFunction::Monotonicity is_monotonic = { .is_monotonic = true }; + const IFunction::Monotonicity is_not_monotonic; + + const DateLUTImpl * date_lut = &DateLUT::instance(); + if (const auto * timezone = dynamic_cast(&type)) + date_lut = &timezone->getTimeZone(); + + if (left.isNull() || right.isNull()) + return is_not_monotonic; + + const auto * type_ptr = &type; + + if (const auto * nullable_type = checkAndGetDataType(type_ptr)) + type_ptr = nullable_type->getNestedType().get(); + + /// The function is monotonous on the [left, right] segment, if the factor transformation returns the same values for them. + + if (checkAndGetDataType(type_ptr)) + { + return Transform::FactorTransform::execute(UInt16(left.get()), *date_lut) + == Transform::FactorTransform::execute(UInt16(right.get()), *date_lut) + ? is_monotonic : is_not_monotonic; + } + else if (checkAndGetDataType(type_ptr)) + { + return Transform::FactorTransform::execute(Int32(left.get()), *date_lut) + == Transform::FactorTransform::execute(Int32(right.get()), *date_lut) + ? is_monotonic : is_not_monotonic; + } + else if (checkAndGetDataType(type_ptr)) + { + return Transform::FactorTransform::execute(UInt32(left.get()), *date_lut) + == Transform::FactorTransform::execute(UInt32(right.get()), *date_lut) + ? is_monotonic : is_not_monotonic; + } + else + { + assert(checkAndGetDataType(type_ptr)); + + const auto & left_date_time = left.get(); + TransformDateTime64 transformer_left(left_date_time.getScale()); + + const auto & right_date_time = right.get(); + TransformDateTime64 transformer_right(right_date_time.getScale()); + + return transformer_left.execute(left_date_time.getValue(), *date_lut) + == transformer_right.execute(right_date_time.getValue(), *date_lut) + ? is_monotonic : is_not_monotonic; + } } } diff --git a/src/Functions/URL/decodeURLComponent.cpp b/src/Functions/URL/decodeURLComponent.cpp index 9e516e73e3c..05e3fbea3fd 100644 --- a/src/Functions/URL/decodeURLComponent.cpp +++ b/src/Functions/URL/decodeURLComponent.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include @@ -14,28 +14,33 @@ namespace ErrorCodes static size_t encodeURL(const char * __restrict src, size_t src_size, char * __restrict dst, bool space_as_plus) { char * dst_pos = dst; - for (size_t i = 0; i < src_size; i++) + for (size_t i = 0; i < src_size; ++i) { if ((src[i] >= '0' && src[i] <= '9') || (src[i] >= 'a' && src[i] <= 'z') || (src[i] >= 'A' && src[i] <= 'Z') || src[i] == '-' || src[i] == '_' || src[i] == '.' || src[i] == '~') { - *dst_pos++ = src[i]; + *dst_pos = src[i]; + ++dst_pos; } else if (src[i] == ' ' && space_as_plus) { - *dst_pos++ = '+'; + *dst_pos = '+'; + ++dst_pos; } else { - *dst_pos++ = '%'; - *dst_pos++ = hexDigitUppercase(src[i] >> 4); - *dst_pos++ = hexDigitUppercase(src[i] & 0xf); + dst_pos[0] = '%'; + ++dst_pos; + writeHexByteUppercase(src[i], dst_pos); + dst_pos += 2; } } - *dst_pos++ = src[src_size]; + *dst_pos = 0; + ++dst_pos; return dst_pos - dst; } + /// We assume that size of the dst buf isn't less than src_size. static size_t decodeURL(const char * __restrict src, size_t src_size, char * __restrict dst, bool plus_as_space) { @@ -120,10 +125,14 @@ struct CodeURLComponentImpl ColumnString::Chars & res_data, ColumnString::Offsets & res_offsets) { if (code_strategy == encode) - //the destination(res_data) string is at most three times the length of the source string + { + /// the destination(res_data) string is at most three times the length of the source string res_data.resize(data.size() * 3); + } else + { res_data.resize(data.size()); + } size_t size = offsets.size(); res_offsets.resize(size); diff --git a/src/Functions/UserDefined/UserDefinedSQLObjectsBackup.cpp b/src/Functions/UserDefined/UserDefinedSQLObjectsBackup.cpp index 6962c21280d..25c309aeb65 100644 --- a/src/Functions/UserDefined/UserDefinedSQLObjectsBackup.cpp +++ b/src/Functions/UserDefined/UserDefinedSQLObjectsBackup.cpp @@ -71,8 +71,7 @@ restoreUserDefinedSQLObjects(RestorerFromBackup & restorer, const String & data_ String function_name = unescapeForFileName(escaped_function_name); String filepath = data_path_in_backup_fs / filename; - auto backup_entry = backup->readFile(filepath); - auto in = backup_entry->getReadBuffer(); + auto in = backup->readFile(filepath); String statement_def; readStringUntilEOF(statement_def, *in); diff --git a/src/Functions/array/arrayIntersect.cpp b/src/Functions/array/arrayIntersect.cpp index 1a718595a3b..c6f0a5afa62 100644 --- a/src/Functions/array/arrayIntersect.cpp +++ b/src/Functions/array/arrayIntersect.cpp @@ -107,7 +107,7 @@ private: : arrays(arrays_), data_type(data_type_), result(result_) {} template - void operator()(Id); + void operator()(TypeList); }; struct DecimalExecutor @@ -120,7 +120,7 @@ private: : arrays(arrays_), data_type(data_type_), result(result_) {} template - void operator()(Id); + void operator()(TypeList); }; }; @@ -446,7 +446,7 @@ ColumnPtr FunctionArrayIntersect::executeImpl(const ColumnsWithTypeAndName & arg } template -void FunctionArrayIntersect::NumberExecutor::operator()(Id) +void FunctionArrayIntersect::NumberExecutor::operator()(TypeList) { using Container = ClearableHashMapWithStackMemory, INITIAL_SIZE_DEGREE>; @@ -456,7 +456,7 @@ void FunctionArrayIntersect::NumberExecutor::operator()(Id) } template -void FunctionArrayIntersect::DecimalExecutor::operator()(Id) +void FunctionArrayIntersect::DecimalExecutor::operator()(TypeList) { using Container = ClearableHashMapWithStackMemory, INITIAL_SIZE_DEGREE>; diff --git a/src/Functions/array/arrayReverse.cpp b/src/Functions/array/arrayReverse.cpp index 912adbadc7c..654a6c4cebf 100644 --- a/src/Functions/array/arrayReverse.cpp +++ b/src/Functions/array/arrayReverse.cpp @@ -91,6 +91,8 @@ ColumnPtr FunctionArrayReverse::executeImpl(const ColumnsWithTypeAndName & argum || executeFixedString(*src_inner_col, offsets, *res_inner_col) || executeGeneric(*src_inner_col, offsets, *res_inner_col); + chassert(bool(src_nullable_col) == bool(res_nullable_col)); + if (src_nullable_col) if (!executeNumber(src_nullable_col->getNullMapColumn(), offsets, res_nullable_col->getNullMapColumn())) throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column {} of null map of the first argument of function {}", diff --git a/src/Functions/bitShiftRight.cpp b/src/Functions/bitShiftRight.cpp index 108847f13ed..21a0f7584aa 100644 --- a/src/Functions/bitShiftRight.cpp +++ b/src/Functions/bitShiftRight.cpp @@ -1,6 +1,6 @@ #include #include -#include +#include namespace DB { diff --git a/src/Functions/bitSlice.cpp b/src/Functions/bitSlice.cpp index 9b0ee4d5f1e..e2b455846d8 100644 --- a/src/Functions/bitSlice.cpp +++ b/src/Functions/bitSlice.cpp @@ -291,7 +291,7 @@ public: ssize_t remain_byte = src.getElementSize() - offset_byte; if (length < 0) { - length_byte = std::max(remain_byte + (length / word_size), static_cast(0)); + length_byte = std::max(remain_byte + (length / word_size), 0z); over_bit = word_size + (length % word_size); if (length_byte == 1 && over_bit <= offset_bit) // begin and end are in same byte AND there are no gaps length_byte = 0; @@ -330,7 +330,7 @@ public: size_t size = src.getElementSize(); if (length < 0) { - length_byte = std::max(static_cast(offset_byte) + (length / word_size), static_cast(0)); + length_byte = std::max(static_cast(offset_byte) + (length / word_size), 0z); over_bit = word_size + (length % word_size); if (length_byte == 1 && over_bit <= offset_bit) // begin and end are in same byte AND there are no gaps length_byte = 0; @@ -395,7 +395,7 @@ public: } else { - length_byte = std::max(remain_byte + (static_cast(length) / word_size), static_cast(0)); + length_byte = std::max(remain_byte + (static_cast(length) / word_size), 0z); over_bit = word_size + (length % word_size); if (length_byte == 1 && over_bit <= offset_bit) // begin and end are in same byte AND there are no gaps length_byte = 0; diff --git a/src/Functions/decodeXMLComponent.cpp b/src/Functions/decodeXMLComponent.cpp index 8b84bb1194e..a25e67e0e37 100644 --- a/src/Functions/decodeXMLComponent.cpp +++ b/src/Functions/decodeXMLComponent.cpp @@ -2,7 +2,7 @@ #include #include #include -#include +#include #include diff --git a/src/Functions/positionCaseInsensitive.cpp b/src/Functions/positionCaseInsensitive.cpp index 4e3b670fe54..f71ce0078cc 100644 --- a/src/Functions/positionCaseInsensitive.cpp +++ b/src/Functions/positionCaseInsensitive.cpp @@ -20,5 +20,6 @@ using FunctionPositionCaseInsensitive = FunctionsStringSearch(); + factory.registerAlias("instr", NamePositionCaseInsensitive::name, FunctionFactory::CaseInsensitive); } } diff --git a/src/Functions/repeat.cpp b/src/Functions/repeat.cpp index dcd05f373fc..0c323c39969 100644 --- a/src/Functions/repeat.cpp +++ b/src/Functions/repeat.cpp @@ -39,13 +39,15 @@ struct RepeatImpl size, max_string_size); } + template static void vectorStrConstRepeat( const ColumnString::Chars & data, const ColumnString::Offsets & offsets, ColumnString::Chars & res_data, ColumnString::Offsets & res_offsets, - UInt64 repeat_time) + T repeat_time) { + repeat_time = repeat_time < 0 ? 0 : repeat_time; checkRepeatTime(repeat_time); UInt64 data_size = 0; @@ -77,7 +79,8 @@ struct RepeatImpl res_offsets.assign(offsets); for (UInt64 i = 0; i < col_num.size(); ++i) { - size_t repeated_size = (offsets[i] - offsets[i - 1] - 1) * col_num[i] + 1; + T repeat_time = col_num[i] < 0 ? 0 : col_num[i]; + size_t repeated_size = (offsets[i] - offsets[i - 1] - 1) * repeat_time + 1; checkStringSize(repeated_size); data_size += repeated_size; res_offsets[i] = data_size; @@ -86,7 +89,7 @@ struct RepeatImpl for (UInt64 i = 0; i < col_num.size(); ++i) { - T repeat_time = col_num[i]; + T repeat_time = col_num[i] < 0 ? 0 : col_num[i]; checkRepeatTime(repeat_time); process(data.data() + offsets[i - 1], res_data.data() + res_offsets[i - 1], offsets[i] - offsets[i - 1], repeat_time); } @@ -105,7 +108,8 @@ struct RepeatImpl UInt64 col_size = col_num.size(); for (UInt64 i = 0; i < col_size; ++i) { - size_t repeated_size = str_size * col_num[i] + 1; + T repeat_time = col_num[i] < 0 ? 0 : col_num[i]; + size_t repeated_size = str_size * repeat_time + 1; checkStringSize(repeated_size); data_size += repeated_size; res_offsets[i] = data_size; @@ -113,7 +117,7 @@ struct RepeatImpl res_data.resize(data_size); for (UInt64 i = 0; i < col_size; ++i) { - T repeat_time = col_num[i]; + T repeat_time = col_num[i] < 0 ? 0 : col_num[i]; checkRepeatTime(repeat_time); process( reinterpret_cast(const_cast(copy_str.data())), @@ -168,7 +172,8 @@ class FunctionRepeat : public IFunction template static bool castType(const IDataType * type, F && f) { - return castTypeToEither(type, std::forward(f)); + return castTypeToEither(type, std::forward(f)); } public: @@ -186,7 +191,7 @@ public: if (!isString(arguments[0])) throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of argument of function {}", arguments[0]->getName(), getName()); - if (!isUnsignedInteger(arguments[1])) + if (!isInteger(arguments[1])) throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of argument of function {}", arguments[1]->getName(), getName()); return arguments[0]; @@ -204,9 +209,15 @@ public: { if (const ColumnConst * scale_column_num = checkAndGetColumn(numcolumn.get())) { - UInt64 repeat_time = scale_column_num->getValue(); auto col_res = ColumnString::create(); - RepeatImpl::vectorStrConstRepeat(col->getChars(), col->getOffsets(), col_res->getChars(), col_res->getOffsets(), repeat_time); + castType(arguments[1].type.get(), [&](const auto & type) + { + using DataType = std::decay_t; + using T = typename DataType::FieldType; + T repeat_time = scale_column_num->getValue(); + RepeatImpl::vectorStrConstRepeat(col->getChars(), col->getOffsets(), col_res->getChars(), col_res->getOffsets(), repeat_time); + return true; + }); return col_res; } else if (castType(arguments[1].type.get(), [&](const auto & type) diff --git a/src/Functions/runningDifference.h b/src/Functions/runningDifference.h index 154370d4cd9..f1ec4f9e523 100644 --- a/src/Functions/runningDifference.h +++ b/src/Functions/runningDifference.h @@ -70,7 +70,7 @@ private: if (!has_prev_value) { - dst[i] = is_first_line_zero ? 0 : src[i]; + dst[i] = is_first_line_zero ? static_cast(0) : static_cast(src[i]); prev = src[i]; has_prev_value = true; } @@ -102,6 +102,10 @@ private: f(UInt32()); else if (which.isUInt64()) f(UInt64()); + else if (which.isUInt128()) + f(UInt128()); + else if (which.isUInt256()) + f(UInt256()); else if (which.isInt8()) f(Int8()); else if (which.isInt16()) @@ -110,6 +114,10 @@ private: f(Int32()); else if (which.isInt64()) f(Int64()); + else if (which.isInt128()) + f(Int128()); + else if (which.isInt256()) + f(Int256()); else if (which.isFloat32()) f(Float32()); else if (which.isFloat64()) diff --git a/src/IO/BackupIOThreadPool.cpp b/src/IO/BackupIOThreadPool.cpp new file mode 100644 index 00000000000..067fc54b1ae --- /dev/null +++ b/src/IO/BackupIOThreadPool.cpp @@ -0,0 +1,34 @@ +#include +#include "Core/Field.h" + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + +std::unique_ptr BackupsIOThreadPool::instance; + +void BackupsIOThreadPool::initialize(size_t max_threads, size_t max_free_threads, size_t queue_size) +{ + if (instance) + { + throw Exception(ErrorCodes::LOGICAL_ERROR, "The BackupsIO thread pool is initialized twice"); + } + + instance = std::make_unique(max_threads, max_free_threads, queue_size, false /*shutdown_on_exception*/); +} + +ThreadPool & BackupsIOThreadPool::get() +{ + if (!instance) + { + throw Exception(ErrorCodes::LOGICAL_ERROR, "The BackupsIO thread pool is not initialized"); + } + + return *instance; +} + +} diff --git a/src/IO/BackupsIOThreadPool.h b/src/IO/BackupsIOThreadPool.h new file mode 100644 index 00000000000..f5aae5741a8 --- /dev/null +++ b/src/IO/BackupsIOThreadPool.h @@ -0,0 +1,20 @@ +#pragma once + +#include + +namespace DB +{ + +/* + * ThreadPool used for the Backup IO. + */ +class BackupsIOThreadPool +{ + static std::unique_ptr instance; + +public: + static void initialize(size_t max_threads, size_t max_free_threads, size_t queue_size); + static ThreadPool & get(); +}; + +} diff --git a/src/IO/HTTPChunkedReadBuffer.cpp b/src/IO/HTTPChunkedReadBuffer.cpp index b9c42088c41..65ccad4aab7 100644 --- a/src/IO/HTTPChunkedReadBuffer.cpp +++ b/src/IO/HTTPChunkedReadBuffer.cpp @@ -2,7 +2,7 @@ #include #include -#include +#include #include diff --git a/src/IO/LimitReadBuffer.cpp b/src/IO/LimitReadBuffer.cpp index 6b3c383c753..e14112f8d19 100644 --- a/src/IO/LimitReadBuffer.cpp +++ b/src/IO/LimitReadBuffer.cpp @@ -9,6 +9,7 @@ namespace DB namespace ErrorCodes { extern const int LIMIT_EXCEEDED; + extern const int CANNOT_READ_ALL_DATA; } @@ -21,14 +22,22 @@ bool LimitReadBuffer::nextImpl() if (bytes >= limit) { + if (exact_limit && bytes == *exact_limit) + return false; + + if (exact_limit && bytes != *exact_limit) + throw Exception(ErrorCodes::CANNOT_READ_ALL_DATA, "Unexpected data, got {} bytes, expected {}", bytes, *exact_limit); + if (throw_exception) throw Exception(ErrorCodes::LIMIT_EXCEEDED, "Limit for LimitReadBuffer exceeded: {}", exception_message); - else - return false; + + return false; } if (!in->next()) { + if (exact_limit && bytes != *exact_limit) + throw Exception(ErrorCodes::CANNOT_READ_ALL_DATA, "Unexpected EOF, got {} of {} bytes", bytes, *exact_limit); /// Clearing the buffer with existing data. set(in->position(), 0); return false; @@ -43,12 +52,14 @@ bool LimitReadBuffer::nextImpl() } -LimitReadBuffer::LimitReadBuffer(ReadBuffer * in_, bool owns, UInt64 limit_, bool throw_exception_, std::string exception_message_) +LimitReadBuffer::LimitReadBuffer(ReadBuffer * in_, bool owns, UInt64 limit_, bool throw_exception_, + std::optional exact_limit_, std::string exception_message_) : ReadBuffer(in_ ? in_->position() : nullptr, 0) , in(in_) , owns_in(owns) , limit(limit_) , throw_exception(throw_exception_) + , exact_limit(exact_limit_) , exception_message(std::move(exception_message_)) { assert(in); @@ -61,14 +72,16 @@ LimitReadBuffer::LimitReadBuffer(ReadBuffer * in_, bool owns, UInt64 limit_, boo } -LimitReadBuffer::LimitReadBuffer(ReadBuffer & in_, UInt64 limit_, bool throw_exception_, std::string exception_message_) - : LimitReadBuffer(&in_, false, limit_, throw_exception_, exception_message_) +LimitReadBuffer::LimitReadBuffer(ReadBuffer & in_, UInt64 limit_, bool throw_exception_, + std::optional exact_limit_, std::string exception_message_) + : LimitReadBuffer(&in_, false, limit_, throw_exception_, exact_limit_, exception_message_) { } -LimitReadBuffer::LimitReadBuffer(std::unique_ptr in_, UInt64 limit_, bool throw_exception_, std::string exception_message_) - : LimitReadBuffer(in_.release(), true, limit_, throw_exception_, exception_message_) +LimitReadBuffer::LimitReadBuffer(std::unique_ptr in_, UInt64 limit_, bool throw_exception_, + std::optional exact_limit_, std::string exception_message_) + : LimitReadBuffer(in_.release(), true, limit_, throw_exception_, exact_limit_, exception_message_) { } diff --git a/src/IO/LimitReadBuffer.h b/src/IO/LimitReadBuffer.h index 92e5fbb0aa6..15885c1d850 100644 --- a/src/IO/LimitReadBuffer.h +++ b/src/IO/LimitReadBuffer.h @@ -13,8 +13,10 @@ namespace DB class LimitReadBuffer : public ReadBuffer { public: - LimitReadBuffer(ReadBuffer & in_, UInt64 limit_, bool throw_exception_, std::string exception_message_ = {}); - LimitReadBuffer(std::unique_ptr in_, UInt64 limit_, bool throw_exception_, std::string exception_message_ = {}); + LimitReadBuffer(ReadBuffer & in_, UInt64 limit_, bool throw_exception_, + std::optional exact_limit_, std::string exception_message_ = {}); + LimitReadBuffer(std::unique_ptr in_, UInt64 limit_, bool throw_exception_, std::optional exact_limit_, + std::string exception_message_ = {}); ~LimitReadBuffer() override; private: @@ -23,9 +25,10 @@ private: UInt64 limit; bool throw_exception; + std::optional exact_limit; std::string exception_message; - LimitReadBuffer(ReadBuffer * in_, bool owns, UInt64 limit_, bool throw_exception_, std::string exception_message_); + LimitReadBuffer(ReadBuffer * in_, bool owns, UInt64 limit_, bool throw_exception_, std::optional exact_limit_, std::string exception_message_); bool nextImpl() override; }; diff --git a/src/IO/MemoryReadWriteBuffer.cpp b/src/IO/MemoryReadWriteBuffer.cpp index 8958390fe03..93ce5ce7ce9 100644 --- a/src/IO/MemoryReadWriteBuffer.cpp +++ b/src/IO/MemoryReadWriteBuffer.cpp @@ -106,7 +106,7 @@ void MemoryWriteBuffer::addChunk() } else { - next_chunk_size = std::max(static_cast(1), static_cast(chunk_tail->size() * growth_rate)); + next_chunk_size = std::max(1uz, static_cast(chunk_tail->size() * growth_rate)); next_chunk_size = std::min(next_chunk_size, max_chunk_size); } diff --git a/src/IO/Operators.h b/src/IO/Operators.h index 06ff20c43e8..185745e8415 100644 --- a/src/IO/Operators.h +++ b/src/IO/Operators.h @@ -30,11 +30,13 @@ enum EscapeManip { escape }; /// For strings - escape special c enum QuoteManip { quote }; /// For strings, dates, datetimes - enclose in single quotes with escaping. In the rest, as usual. enum DoubleQuoteManip { double_quote }; /// For strings, dates, datetimes - enclose in double quotes with escaping. In the rest, as usual. enum BinaryManip { binary }; /// Output in binary format. +enum XMLManip { xml }; /// Output strings with XML escaping. struct EscapeManipWriteBuffer : std::reference_wrapper { using std::reference_wrapper::reference_wrapper; }; struct QuoteManipWriteBuffer : std::reference_wrapper { using std::reference_wrapper::reference_wrapper; }; struct DoubleQuoteManipWriteBuffer : std::reference_wrapper { using std::reference_wrapper::reference_wrapper; }; struct BinaryManipWriteBuffer : std::reference_wrapper { using std::reference_wrapper::reference_wrapper; }; +struct XMLManipWriteBuffer : std::reference_wrapper { using std::reference_wrapper::reference_wrapper; }; struct EscapeManipReadBuffer : std::reference_wrapper { using std::reference_wrapper::reference_wrapper; }; struct QuoteManipReadBuffer : std::reference_wrapper { using std::reference_wrapper::reference_wrapper; }; @@ -48,11 +50,13 @@ inline EscapeManipWriteBuffer operator<< (WriteBuffer & buf, EscapeManip) inline QuoteManipWriteBuffer operator<< (WriteBuffer & buf, QuoteManip) { return buf; } inline DoubleQuoteManipWriteBuffer operator<< (WriteBuffer & buf, DoubleQuoteManip) { return buf; } inline BinaryManipWriteBuffer operator<< (WriteBuffer & buf, BinaryManip) { return buf; } +inline XMLManipWriteBuffer operator<< (WriteBuffer & buf, XMLManip) { return buf; } template WriteBuffer & operator<< (EscapeManipWriteBuffer buf, const T & x) { writeText(x, buf.get()); return buf; } template WriteBuffer & operator<< (QuoteManipWriteBuffer buf, const T & x) { writeQuoted(x, buf.get()); return buf; } template WriteBuffer & operator<< (DoubleQuoteManipWriteBuffer buf, const T & x) { writeDoubleQuoted(x, buf.get()); return buf; } template WriteBuffer & operator<< (BinaryManipWriteBuffer buf, const T & x) { writeBinary(x, buf.get()); return buf; } +template WriteBuffer & operator<< (XMLManipWriteBuffer buf, const T & x) { writeText(x, buf.get()); return buf; } inline WriteBuffer & operator<< (EscapeManipWriteBuffer buf, const String & x) { writeEscapedString(x, buf); return buf; } inline WriteBuffer & operator<< (EscapeManipWriteBuffer buf, std::string_view x) { writeEscapedString(x, buf); return buf; } @@ -63,6 +67,10 @@ inline WriteBuffer & operator<< (QuoteManipWriteBuffer buf, const char * x inline WriteBuffer & operator<< (DoubleQuoteManipWriteBuffer buf, const char * x) { writeAnyQuotedString<'"'>(x, x + strlen(x), buf.get()); return buf; } inline WriteBuffer & operator<< (BinaryManipWriteBuffer buf, const char * x) { writeStringBinary(x, buf.get()); return buf; } +inline WriteBuffer & operator<< (XMLManipWriteBuffer buf, std::string_view x) { writeXMLStringForTextElementOrAttributeValue(x, buf); return buf; } +inline WriteBuffer & operator<< (XMLManipWriteBuffer buf, StringRef x) { writeXMLStringForTextElementOrAttributeValue(x.toView(), buf); return buf; } +inline WriteBuffer & operator<< (XMLManipWriteBuffer buf, const char * x) { writeXMLStringForTextElementOrAttributeValue(std::string_view(x), buf); return buf; } + /// The manipulator calls the WriteBuffer method `next` - this makes the buffer reset. For nested buffers, the reset is not recursive. enum FlushManip { flush }; diff --git a/src/IO/ReadHelpers.cpp b/src/IO/ReadHelpers.cpp index 86a2b9c650e..e14b3ae9129 100644 --- a/src/IO/ReadHelpers.cpp +++ b/src/IO/ReadHelpers.cpp @@ -1,5 +1,5 @@ #include -#include +#include #include #include #include diff --git a/src/IO/ReadWriteBufferFromHTTP.h b/src/IO/ReadWriteBufferFromHTTP.h index 5c5285c5260..ed25c101d7e 100644 --- a/src/IO/ReadWriteBufferFromHTTP.h +++ b/src/IO/ReadWriteBufferFromHTTP.h @@ -259,12 +259,12 @@ namespace detail { try { - callWithRedirects(response, Poco::Net::HTTPRequest::HTTP_HEAD); + callWithRedirects(response, Poco::Net::HTTPRequest::HTTP_HEAD, true); break; } catch (const Poco::Exception & e) { - if (i == settings.http_max_tries - 1) + if (i == settings.http_max_tries - 1 || !isRetriableError(response.getStatus())) throw; LOG_ERROR(log, "Failed to make HTTP_HEAD request to {}. Error: {}", uri.toString(), e.displayText()); @@ -353,11 +353,12 @@ namespace detail static bool isRetriableError(const Poco::Net::HTTPResponse::HTTPStatus http_status) noexcept { - constexpr std::array non_retriable_errors{ + static constexpr std::array non_retriable_errors{ Poco::Net::HTTPResponse::HTTPStatus::HTTP_BAD_REQUEST, Poco::Net::HTTPResponse::HTTPStatus::HTTP_UNAUTHORIZED, Poco::Net::HTTPResponse::HTTPStatus::HTTP_NOT_FOUND, Poco::Net::HTTPResponse::HTTPStatus::HTTP_FORBIDDEN, + Poco::Net::HTTPResponse::HTTPStatus::HTTP_NOT_IMPLEMENTED, Poco::Net::HTTPResponse::HTTPStatus::HTTP_METHOD_NOT_ALLOWED}; return std::all_of( diff --git a/src/IO/S3/Client.cpp b/src/IO/S3/Client.cpp index 5c0539ee486..aba884948da 100644 --- a/src/IO/S3/Client.cpp +++ b/src/IO/S3/Client.cpp @@ -123,9 +123,8 @@ Client::Client( { auto * endpoint_provider = dynamic_cast(accessEndpointProvider().get()); endpoint_provider->GetBuiltInParameters().GetParameter("Region").GetString(explicit_region); - std::string endpoint; - endpoint_provider->GetBuiltInParameters().GetParameter("Endpoint").GetString(endpoint); - detect_region = explicit_region == Aws::Region::AWS_GLOBAL && endpoint.find(".amazonaws.com") != std::string::npos; + endpoint_provider->GetBuiltInParameters().GetParameter("Endpoint").GetString(initial_endpoint); + detect_region = explicit_region == Aws::Region::AWS_GLOBAL && initial_endpoint.find(".amazonaws.com") != std::string::npos; cache = std::make_shared(); ClientCacheRegistry::instance().registerClient(cache); @@ -133,6 +132,7 @@ Client::Client( Client::Client(const Client & other) : Aws::S3::S3Client(other) + , initial_endpoint(other.initial_endpoint) , explicit_region(other.explicit_region) , detect_region(other.detect_region) , max_redirects(other.max_redirects) diff --git a/src/IO/S3/Client.h b/src/IO/S3/Client.h index 18ba62d1006..7ac97555dd1 100644 --- a/src/IO/S3/Client.h +++ b/src/IO/S3/Client.h @@ -109,6 +109,9 @@ public: } } + /// Returns the initial endpoint. + const String & getInitialEndpoint() const { return initial_endpoint; } + /// Decorator for RetryStrategy needed for this client to work correctly. /// We want to manually handle permanent moves (status code 301) because: /// - redirect location is written in XML format inside the response body something that doesn't exist for HEAD @@ -198,6 +201,8 @@ private: bool checkIfWrongRegionDefined(const std::string & bucket, const Aws::S3::S3Error & error, std::string & region) const; void insertRegionOverride(const std::string & bucket, const std::string & region) const; + String initial_endpoint; + std::string explicit_region; mutable bool detect_region = true; diff --git a/src/IO/WriteBufferFromHTTP.cpp b/src/IO/WriteBufferFromHTTP.cpp index f7456ad6b6c..355c42a23c9 100644 --- a/src/IO/WriteBufferFromHTTP.cpp +++ b/src/IO/WriteBufferFromHTTP.cpp @@ -11,6 +11,7 @@ WriteBufferFromHTTP::WriteBufferFromHTTP( const std::string & method, const std::string & content_type, const std::string & content_encoding, + const HTTPHeaderEntries & additional_headers, const ConnectionTimeouts & timeouts, size_t buffer_size_) : WriteBufferFromOStream(buffer_size_) @@ -28,6 +29,9 @@ WriteBufferFromHTTP::WriteBufferFromHTTP( if (!content_encoding.empty()) request.set("Content-Encoding", content_encoding); + for (const auto & header: additional_headers) + request.add(header.name, header.value); + LOG_TRACE((&Poco::Logger::get("WriteBufferToHTTP")), "Sending request to {}", uri.toString()); ostr = &session->sendRequest(request); diff --git a/src/IO/WriteBufferFromHTTP.h b/src/IO/WriteBufferFromHTTP.h index 6966bc8a5c5..ce5020dfa78 100644 --- a/src/IO/WriteBufferFromHTTP.h +++ b/src/IO/WriteBufferFromHTTP.h @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -22,6 +23,7 @@ public: const std::string & method = Poco::Net::HTTPRequest::HTTP_POST, // POST or PUT only const std::string & content_type = "", const std::string & content_encoding = "", + const HTTPHeaderEntries & additional_headers = {}, const ConnectionTimeouts & timeouts = {}, size_t buffer_size_ = DBMS_DEFAULT_BUFFER_SIZE); diff --git a/src/IO/WriteBufferFromPocoSocket.cpp b/src/IO/WriteBufferFromPocoSocket.cpp index e01245849ae..b15149c6f88 100644 --- a/src/IO/WriteBufferFromPocoSocket.cpp +++ b/src/IO/WriteBufferFromPocoSocket.cpp @@ -64,7 +64,8 @@ void WriteBufferFromPocoSocket::nextImpl() } catch (const Poco::Net::NetException & e) { - throw NetException(ErrorCodes::NETWORK_ERROR, "{}, while writing to socket ({})", e.displayText(), peer_address.toString()); + throw NetException(ErrorCodes::NETWORK_ERROR, "{}, while writing to socket ({} -> {})", e.displayText(), + our_address.toString(), peer_address.toString()); } catch (const Poco::TimeoutException &) { @@ -74,18 +75,20 @@ void WriteBufferFromPocoSocket::nextImpl() } catch (const Poco::IOException & e) { - throw NetException(ErrorCodes::NETWORK_ERROR, "{}, while writing to socket ({})", e.displayText(), peer_address.toString()); + throw NetException(ErrorCodes::NETWORK_ERROR, "{}, while writing to socket ({} -> {})", e.displayText(), + our_address.toString(), peer_address.toString()); } if (res < 0) - throw NetException(ErrorCodes::CANNOT_WRITE_TO_SOCKET, "Cannot write to socket ({})", peer_address.toString()); + throw NetException(ErrorCodes::CANNOT_WRITE_TO_SOCKET, "Cannot write to socket ({} -> {})", + our_address.toString(), peer_address.toString()); bytes_written += res; } } WriteBufferFromPocoSocket::WriteBufferFromPocoSocket(Poco::Net::Socket & socket_, size_t buf_size) - : BufferWithOwnMemory(buf_size), socket(socket_), peer_address(socket.peerAddress()) + : BufferWithOwnMemory(buf_size), socket(socket_), peer_address(socket.peerAddress()), our_address(socket.address()) { } diff --git a/src/IO/WriteBufferFromPocoSocket.h b/src/IO/WriteBufferFromPocoSocket.h index 295ca16ecaf..ffe1176c8cd 100644 --- a/src/IO/WriteBufferFromPocoSocket.h +++ b/src/IO/WriteBufferFromPocoSocket.h @@ -28,6 +28,7 @@ protected: * (getpeername will return an error). */ Poco::Net::SocketAddress peer_address; + Poco::Net::SocketAddress our_address; }; } diff --git a/src/IO/WriteHelpers.cpp b/src/IO/WriteHelpers.cpp index caeea0a82a3..a0eceddc6f6 100644 --- a/src/IO/WriteHelpers.cpp +++ b/src/IO/WriteHelpers.cpp @@ -2,7 +2,7 @@ #include #include #include -#include +#include namespace DB diff --git a/src/IO/copyData.cpp b/src/IO/copyData.cpp index b189c318d67..07222a930b5 100644 --- a/src/IO/copyData.cpp +++ b/src/IO/copyData.cpp @@ -10,6 +10,7 @@ namespace DB namespace ErrorCodes { extern const int ATTEMPT_TO_READ_AFTER_EOF; + extern const int CANNOT_READ_ALL_DATA; } namespace @@ -91,6 +92,13 @@ void copyData(ReadBuffer & from, WriteBuffer & to, size_t bytes, std::function & is_cancelled, ThrottlerPtr throttler) { copyDataImpl(from, to, false, std::numeric_limits::max(), &is_cancelled, throttler); diff --git a/src/IO/copyData.h b/src/IO/copyData.h index 2202f36f79e..b67088d8e47 100644 --- a/src/IO/copyData.h +++ b/src/IO/copyData.h @@ -27,6 +27,9 @@ void copyData(ReadBuffer & from, WriteBuffer & to, size_t bytes, const std::atom void copyData(ReadBuffer & from, WriteBuffer & to, std::function cancellation_hook); void copyData(ReadBuffer & from, WriteBuffer & to, size_t bytes, std::function cancellation_hook); +/// Copies at most `max_bytes` bytes from ReadBuffer to WriteBuffer. If there are more bytes, then throws an exception. +void copyDataMaxBytes(ReadBuffer & from, WriteBuffer & to, size_t max_bytes); + /// Same as above but also use throttler to limit maximum speed void copyDataWithThrottler(ReadBuffer & from, WriteBuffer & to, const std::atomic & is_cancelled, ThrottlerPtr throttler); void copyDataWithThrottler(ReadBuffer & from, WriteBuffer & to, size_t bytes, const std::atomic & is_cancelled, ThrottlerPtr throttler); diff --git a/src/IO/examples/limit_read_buffer.cpp b/src/IO/examples/limit_read_buffer.cpp index 559c87353f0..9c57c175620 100644 --- a/src/IO/examples/limit_read_buffer.cpp +++ b/src/IO/examples/limit_read_buffer.cpp @@ -24,13 +24,13 @@ int main(int argc, char ** argv) writeCString("--- first ---\n", out); { - LimitReadBuffer limit_in(in, limit, false); + LimitReadBuffer limit_in(in, limit, /* trow_exception */ false, /* exact_limit */ {}); copyData(limit_in, out); } writeCString("\n--- second ---\n", out); { - LimitReadBuffer limit_in(in, limit, false); + LimitReadBuffer limit_in(in, limit, /* trow_exception */ false, /* exact_limit */ {}); copyData(limit_in, out); } diff --git a/src/IO/examples/limit_read_buffer2.cpp b/src/IO/examples/limit_read_buffer2.cpp index ac7c43d764c..a0369047d3a 100644 --- a/src/IO/examples/limit_read_buffer2.cpp +++ b/src/IO/examples/limit_read_buffer2.cpp @@ -27,7 +27,7 @@ try ReadBuffer in(src.data(), src.size(), 0); - LimitReadBuffer limit_in(in, 1, false); + LimitReadBuffer limit_in(in, 1, /* trow_exception */ false, /* exact_limit */ {}); { WriteBufferFromString out(dst); @@ -55,7 +55,7 @@ try char x; readChar(x, in); - LimitReadBuffer limit_in(in, 1, false); + LimitReadBuffer limit_in(in, 1, /* trow_exception */ false, /* exact_limit */ {}); copyData(limit_in, out); @@ -85,7 +85,7 @@ try ReadBuffer in(src.data(), src.size(), 0); { - LimitReadBuffer limit_in(in, 1, false); + LimitReadBuffer limit_in(in, 1, /* trow_exception */ false, /* exact_limit */ {}); char x; readChar(x, limit_in); diff --git a/src/IO/tests/gtest_hadoop_snappy_decoder.cpp b/src/IO/tests/gtest_hadoop_snappy_decoder.cpp index 4db0deac08e..2847c730735 100644 --- a/src/IO/tests/gtest_hadoop_snappy_decoder.cpp +++ b/src/IO/tests/gtest_hadoop_snappy_decoder.cpp @@ -14,7 +14,7 @@ #include #include #include -#include +#include using namespace DB; TEST(HadoopSnappyDecoder, repeatNeedMoreInput) { diff --git a/src/Interpreters/ActionsDAG.cpp b/src/Interpreters/ActionsDAG.cpp index daba4c1608d..46b5a93b28c 100644 --- a/src/Interpreters/ActionsDAG.cpp +++ b/src/Interpreters/ActionsDAG.cpp @@ -1,5 +1,6 @@ #include +#include #include #include #include @@ -199,6 +200,23 @@ const ActionsDAG::Node & ActionsDAG::addFunction( std::move(children), std::move(arguments), std::move(result_name), + function_base->getResultType(), + all_const); +} + +const ActionsDAG::Node & ActionsDAG::addFunction( + const FunctionNode & function, + NodeRawConstPtrs children, + std::string result_name) +{ + auto [arguments, all_const] = getFunctionArguments(children); + + return addFunctionImpl( + function.getFunction(), + std::move(children), + std::move(arguments), + std::move(result_name), + function.getResultType(), all_const); } @@ -214,6 +232,7 @@ const ActionsDAG::Node & ActionsDAG::addFunction( std::move(children), std::move(arguments), std::move(result_name), + function_base->getResultType(), all_const); } @@ -238,6 +257,7 @@ const ActionsDAG::Node & ActionsDAG::addFunctionImpl( NodeRawConstPtrs children, ColumnsWithTypeAndName arguments, std::string result_name, + DataTypePtr result_type, bool all_const) { size_t num_arguments = children.size(); @@ -247,7 +267,7 @@ const ActionsDAG::Node & ActionsDAG::addFunctionImpl( node.children = std::move(children); node.function_base = function_base; - node.result_type = node.function_base->getResultType(); + node.result_type = result_type; node.function = node.function_base->prepare(arguments); node.is_deterministic = node.function_base->isDeterministic(); @@ -2264,7 +2284,15 @@ ActionsDAGPtr ActionsDAG::buildFilterActionsDAG( for (const auto & child : node->children) function_children.push_back(node_to_result_node.find(child)->second); - result_node = &result_dag->addFunction(node->function_base, std::move(function_children), {}); + auto [arguments, all_const] = getFunctionArguments(function_children); + + result_node = &result_dag->addFunctionImpl( + node->function_base, + std::move(function_children), + std::move(arguments), + {}, + node->result_type, + all_const); break; } } diff --git a/src/Interpreters/ActionsDAG.h b/src/Interpreters/ActionsDAG.h index 0182db8e027..5f0005dae37 100644 --- a/src/Interpreters/ActionsDAG.h +++ b/src/Interpreters/ActionsDAG.h @@ -23,6 +23,8 @@ using FunctionBasePtr = std::shared_ptr; class IFunctionOverloadResolver; using FunctionOverloadResolverPtr = std::shared_ptr; +class FunctionNode; + class IDataType; using DataTypePtr = std::shared_ptr; @@ -139,6 +141,10 @@ public: const FunctionOverloadResolverPtr & function, NodeRawConstPtrs children, std::string result_name); + const Node & addFunction( + const FunctionNode & function, + NodeRawConstPtrs children, + std::string result_name); const Node & addFunction( const FunctionBasePtr & function_base, NodeRawConstPtrs children, @@ -358,6 +364,7 @@ private: NodeRawConstPtrs children, ColumnsWithTypeAndName arguments, std::string result_name, + DataTypePtr result_type, bool all_const); #if USE_EMBEDDED_COMPILER diff --git a/src/Interpreters/Aggregator.cpp b/src/Interpreters/Aggregator.cpp index 229cb818735..09c2eebfdd6 100644 --- a/src/Interpreters/Aggregator.cpp +++ b/src/Interpreters/Aggregator.cpp @@ -1441,6 +1441,11 @@ void Aggregator::prepareAggregateInstructions( materialized_columns.push_back(columns.at(pos)->convertToFullColumnIfConst()); aggregate_columns[i][j] = materialized_columns.back().get(); + /// Sparse columns without defaults may be handled incorrectly. + if (aggregate_columns[i][j]->isSparse() + && aggregate_columns[i][j]->getNumberOfDefaultRows() == 0) + allow_sparse_arguments = false; + auto full_column = allow_sparse_arguments ? aggregate_columns[i][j]->getPtr() : recursiveRemoveSparse(aggregate_columns[i][j]->getPtr()); diff --git a/src/Interpreters/AsynchronousInsertQueue.cpp b/src/Interpreters/AsynchronousInsertQueue.cpp index ac8101d4ca2..590cbc9ba83 100644 --- a/src/Interpreters/AsynchronousInsertQueue.cpp +++ b/src/Interpreters/AsynchronousInsertQueue.cpp @@ -18,7 +18,6 @@ #include #include #include -#include #include #include #include @@ -104,10 +103,9 @@ bool AsynchronousInsertQueue::InsertQuery::operator==(const InsertQuery & other) return query_str == other.query_str && settings == other.settings; } -AsynchronousInsertQueue::InsertData::Entry::Entry(String && bytes_, String && query_id_, MemoryTracker * user_memory_tracker_) +AsynchronousInsertQueue::InsertData::Entry::Entry(String && bytes_, String && query_id_) : bytes(std::move(bytes_)) , query_id(std::move(query_id_)) - , user_memory_tracker(user_memory_tracker_) , create_time(std::chrono::system_clock::now()) { } @@ -210,7 +208,7 @@ AsynchronousInsertQueue::push(ASTPtr query, ContextPtr query_context) /// to avoid buffering of huge amount of data in memory. auto read_buf = getReadBufferFromASTInsertQuery(query); - LimitReadBuffer limit_buf(*read_buf, settings.async_insert_max_data_size, false); + LimitReadBuffer limit_buf(*read_buf, settings.async_insert_max_data_size, /* trow_exception */ false, /* exact_limit */ {}); WriteBufferFromString write_buf(bytes); copyData(limit_buf, write_buf); @@ -236,7 +234,7 @@ AsynchronousInsertQueue::push(ASTPtr query, ContextPtr query_context) if (auto quota = query_context->getQuota()) quota->used(QuotaType::WRITTEN_BYTES, bytes.size()); - auto entry = std::make_shared(std::move(bytes), query_context->getCurrentQueryId(), CurrentThread::getUserMemoryTracker()); + auto entry = std::make_shared(std::move(bytes), query_context->getCurrentQueryId()); InsertQuery key{query, settings}; InsertDataPtr data_to_process; diff --git a/src/Interpreters/AsynchronousInsertQueue.h b/src/Interpreters/AsynchronousInsertQueue.h index e6b7bff8d26..23a2860364d 100644 --- a/src/Interpreters/AsynchronousInsertQueue.h +++ b/src/Interpreters/AsynchronousInsertQueue.h @@ -1,7 +1,6 @@ #pragma once #include -#include #include #include #include @@ -60,31 +59,6 @@ private: UInt128 calculateHash() const; }; - struct UserMemoryTrackerSwitcher - { - explicit UserMemoryTrackerSwitcher(MemoryTracker * new_tracker) - { - auto * thread_tracker = CurrentThread::getMemoryTracker(); - prev_untracked_memory = current_thread->untracked_memory; - prev_memory_tracker_parent = thread_tracker->getParent(); - - current_thread->untracked_memory = 0; - thread_tracker->setParent(new_tracker); - } - - ~UserMemoryTrackerSwitcher() - { - CurrentThread::flushUntrackedMemory(); - auto * thread_tracker = CurrentThread::getMemoryTracker(); - - current_thread->untracked_memory = prev_untracked_memory; - thread_tracker->setParent(prev_memory_tracker_parent); - } - - MemoryTracker * prev_memory_tracker_parent; - Int64 prev_untracked_memory; - }; - struct InsertData { struct Entry @@ -92,10 +66,9 @@ private: public: const String bytes; const String query_id; - MemoryTracker * const user_memory_tracker; const std::chrono::time_point create_time; - Entry(String && bytes_, String && query_id_, MemoryTracker * user_memory_tracker_); + Entry(String && bytes_, String && query_id_); void finish(std::exception_ptr exception_ = nullptr); std::future getFuture() { return promise.get_future(); } @@ -106,19 +79,6 @@ private: std::atomic_bool finished = false; }; - ~InsertData() - { - auto it = entries.begin(); - // Entries must be destroyed in context of user who runs async insert. - // Each entry in the list may correspond to a different user, - // so we need to switch current thread's MemoryTracker parent on each iteration. - while (it != entries.end()) - { - UserMemoryTrackerSwitcher switcher((*it)->user_memory_tracker); - it = entries.erase(it); - } - } - using EntryPtr = std::shared_ptr; std::list entries; diff --git a/src/Interpreters/Cache/FileCacheKey.h b/src/Interpreters/Cache/FileCacheKey.h index fed4c7f47e0..67e1466e2d4 100644 --- a/src/Interpreters/Cache/FileCacheKey.h +++ b/src/Interpreters/Cache/FileCacheKey.h @@ -1,6 +1,6 @@ #pragma once #include -#include +#include #include namespace DB diff --git a/src/Interpreters/Cache/FileSegment.cpp b/src/Interpreters/Cache/FileSegment.cpp index f4d7b2612a5..bd4554c6532 100644 --- a/src/Interpreters/Cache/FileSegment.cpp +++ b/src/Interpreters/Cache/FileSegment.cpp @@ -2,7 +2,7 @@ #include #include -#include +#include #include #include #include diff --git a/src/Interpreters/Cache/LRUFileCachePriority.cpp b/src/Interpreters/Cache/LRUFileCachePriority.cpp index 8010b9c682b..c20379e1fc1 100644 --- a/src/Interpreters/Cache/LRUFileCachePriority.cpp +++ b/src/Interpreters/Cache/LRUFileCachePriority.cpp @@ -34,7 +34,7 @@ IFileCachePriority::WriteIterator LRUFileCachePriority::add(const Key & key, siz CurrentMetrics::add(CurrentMetrics::FilesystemCacheSize, size); CurrentMetrics::add(CurrentMetrics::FilesystemCacheElements); - LOG_TRACE(log, "Added entry into LRU queue, key: {}, offset: {}", key.toString(), offset); + LOG_TEST(log, "Added entry into LRU queue, key: {}, offset: {}", key.toString(), offset); return std::make_shared(this, iter); } @@ -54,7 +54,7 @@ void LRUFileCachePriority::removeAll(std::lock_guard &) CurrentMetrics::sub(CurrentMetrics::FilesystemCacheSize, cache_size); CurrentMetrics::sub(CurrentMetrics::FilesystemCacheElements, queue.size()); - LOG_TRACE(log, "Removed all entries from LRU queue"); + LOG_TEST(log, "Removed all entries from LRU queue"); queue.clear(); cache_size = 0; @@ -88,7 +88,7 @@ void LRUFileCachePriority::LRUFileCacheIterator::removeAndGetNext(std::lock_guar CurrentMetrics::sub(CurrentMetrics::FilesystemCacheSize, queue_iter->size); CurrentMetrics::sub(CurrentMetrics::FilesystemCacheElements); - LOG_TRACE(cache_priority->log, "Removed entry from LRU queue, key: {}, offset: {}", queue_iter->key.toString(), queue_iter->offset); + LOG_TEST(cache_priority->log, "Removed entry from LRU queue, key: {}, offset: {}", queue_iter->key.toString(), queue_iter->offset); queue_iter = cache_priority->queue.erase(queue_iter); } diff --git a/src/Interpreters/Cluster.cpp b/src/Interpreters/Cluster.cpp index bf3a66fed99..0add0e427f9 100644 --- a/src/Interpreters/Cluster.cpp +++ b/src/Interpreters/Cluster.cpp @@ -15,6 +15,7 @@ #include #include +#include namespace DB { @@ -509,7 +510,6 @@ Cluster::Cluster(const Poco::Util::AbstractConfiguration & config, shard_local_addresses.push_back(replica); shard_all_addresses.push_back(replica); } - ConnectionPoolWithFailoverPtr shard_pool = std::make_shared( all_replicas_pools, settings.load_balancing, settings.distributed_replica_error_half_life.totalSeconds(), settings.distributed_replica_error_cap); @@ -653,9 +653,9 @@ void Cluster::initMisc() } } -std::unique_ptr Cluster::getClusterWithReplicasAsShards(const Settings & settings) const +std::unique_ptr Cluster::getClusterWithReplicasAsShards(const Settings & settings, size_t max_replicas_from_shard) const { - return std::unique_ptr{ new Cluster(ReplicasAsShardsTag{}, *this, settings)}; + return std::unique_ptr{ new Cluster(ReplicasAsShardsTag{}, *this, settings, max_replicas_from_shard)}; } std::unique_ptr Cluster::getClusterWithSingleShard(size_t index) const @@ -668,7 +668,44 @@ std::unique_ptr Cluster::getClusterWithMultipleShards(const std::vector return std::unique_ptr{ new Cluster(SubclusterTag{}, *this, indices) }; } -Cluster::Cluster(Cluster::ReplicasAsShardsTag, const Cluster & from, const Settings & settings) +namespace +{ + +void shuffleReplicas(std::vector & replicas, const Settings & settings, size_t replicas_needed) +{ + std::random_device rd; + std::mt19937 gen{rd()}; + + if (settings.prefer_localhost_replica) + { + // force for local replica to always be included + auto first_non_local_replica = std::partition(replicas.begin(), replicas.end(), [](const auto & replica) { return replica.is_local; }); + size_t local_replicas_count = first_non_local_replica - replicas.begin(); + + if (local_replicas_count == replicas_needed) + { + /// we have exact amount of local replicas as needed, no need to do anything + return; + } + + if (local_replicas_count > replicas_needed) + { + /// we can use only local replicas, shuffle them + std::shuffle(replicas.begin(), first_non_local_replica, gen); + return; + } + + /// shuffle just non local replicas + std::shuffle(first_non_local_replica, replicas.end(), gen); + return; + } + + std::shuffle(replicas.begin(), replicas.end(), gen); +} + +} + +Cluster::Cluster(Cluster::ReplicasAsShardsTag, const Cluster & from, const Settings & settings, size_t max_replicas_from_shard) { if (from.addresses_with_failover.empty()) throw Exception(ErrorCodes::LOGICAL_ERROR, "Cluster is empty"); @@ -677,40 +714,55 @@ Cluster::Cluster(Cluster::ReplicasAsShardsTag, const Cluster & from, const Setti std::set> unique_hosts; for (size_t shard_index : collections::range(0, from.shards_info.size())) { - const auto & replicas = from.addresses_with_failover[shard_index]; - for (const auto & address : replicas) + auto create_shards_from_replicas = [&](std::span replicas) { - if (!unique_hosts.emplace(address.host_name, address.port).second) - continue; /// Duplicate host, skip. + for (const auto & address : replicas) + { + if (!unique_hosts.emplace(address.host_name, address.port).second) + continue; /// Duplicate host, skip. - ShardInfo info; - info.shard_num = ++shard_num; + ShardInfo info; + info.shard_num = ++shard_num; - if (address.is_local) - info.local_addresses.push_back(address); + if (address.is_local) + info.local_addresses.push_back(address); - info.all_addresses.push_back(address); + info.all_addresses.push_back(address); - auto pool = ConnectionPoolFactory::instance().get( - static_cast(settings.distributed_connections_pool_size), - address.host_name, - address.port, - address.default_database, - address.user, - address.password, - address.quota_key, - address.cluster, - address.cluster_secret, - "server", - address.compression, - address.secure, - address.priority); + auto pool = ConnectionPoolFactory::instance().get( + static_cast(settings.distributed_connections_pool_size), + address.host_name, + address.port, + address.default_database, + address.user, + address.password, + address.quota_key, + address.cluster, + address.cluster_secret, + "server", + address.compression, + address.secure, + address.priority); - info.pool = std::make_shared(ConnectionPoolPtrs{pool}, settings.load_balancing); - info.per_replica_pools = {std::move(pool)}; + info.pool = std::make_shared(ConnectionPoolPtrs{pool}, settings.load_balancing); + info.per_replica_pools = {std::move(pool)}; - addresses_with_failover.emplace_back(Addresses{address}); - shards_info.emplace_back(std::move(info)); + addresses_with_failover.emplace_back(Addresses{address}); + shards_info.emplace_back(std::move(info)); + } + }; + + const auto & replicas = from.addresses_with_failover[shard_index]; + if (!max_replicas_from_shard || replicas.size() <= max_replicas_from_shard) + { + create_shards_from_replicas(replicas); + } + else + { + auto shuffled_replicas = replicas; + // shuffle replicas so we don't always pick the same subset + shuffleReplicas(shuffled_replicas, settings, max_replicas_from_shard); + create_shards_from_replicas(std::span{shuffled_replicas.begin(), max_replicas_from_shard}); } } diff --git a/src/Interpreters/Cluster.h b/src/Interpreters/Cluster.h index a7f5a914974..02c450c3c6b 100644 --- a/src/Interpreters/Cluster.h +++ b/src/Interpreters/Cluster.h @@ -250,7 +250,7 @@ public: std::unique_ptr getClusterWithMultipleShards(const std::vector & indices) const; /// Get a new Cluster that contains all servers (all shards with all replicas) from existing cluster as independent shards. - std::unique_ptr getClusterWithReplicasAsShards(const Settings & settings) const; + std::unique_ptr getClusterWithReplicasAsShards(const Settings & settings, size_t max_replicas_from_shard = 0) const; /// Returns false if cluster configuration doesn't allow to use it for cross-replication. /// NOTE: true does not mean, that it's actually a cross-replication cluster. @@ -271,7 +271,7 @@ private: /// For getClusterWithReplicasAsShards implementation struct ReplicasAsShardsTag {}; - Cluster(ReplicasAsShardsTag, const Cluster & from, const Settings & settings); + Cluster(ReplicasAsShardsTag, const Cluster & from, const Settings & settings, size_t max_replicas_from_shard); /// Inter-server secret String secret; diff --git a/src/Interpreters/ClusterProxy/executeQuery.cpp b/src/Interpreters/ClusterProxy/executeQuery.cpp index 85a012d126f..439033f89f2 100644 --- a/src/Interpreters/ClusterProxy/executeQuery.cpp +++ b/src/Interpreters/ClusterProxy/executeQuery.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -20,6 +21,7 @@ #include #include + namespace DB { @@ -157,7 +159,8 @@ void executeQuery( const ASTPtr & query_ast, ContextPtr context, const SelectQueryInfo & query_info, const ExpressionActionsPtr & sharding_key_expr, const std::string & sharding_key_column_name, - const ClusterPtr & not_optimized_cluster) + const ClusterPtr & not_optimized_cluster, + AdditionalShardFilterGenerator shard_filter_generator) { const Settings & settings = context->getSettingsRef(); @@ -189,7 +192,22 @@ void executeQuery( visitor.visit(query_ast_for_shard); } else - query_ast_for_shard = query_ast; + query_ast_for_shard = query_ast->clone(); + + if (shard_filter_generator) + { + auto shard_filter = shard_filter_generator(shard_info.shard_num); + if (shard_filter) + { + auto & select_query = query_ast_for_shard->as(); + + auto where_expression = select_query.where(); + if (where_expression) + shard_filter = makeASTFunction("and", where_expression, shard_filter); + + select_query.setExpression(ASTSelectQuery::Expression::WHERE, std::move(shard_filter)); + } + } stream_factory.createForShard(shard_info, query_ast_for_shard, main_table, table_func_ptr, diff --git a/src/Interpreters/ClusterProxy/executeQuery.h b/src/Interpreters/ClusterProxy/executeQuery.h index 787e79313cc..41f6da55686 100644 --- a/src/Interpreters/ClusterProxy/executeQuery.h +++ b/src/Interpreters/ClusterProxy/executeQuery.h @@ -37,6 +37,7 @@ class SelectStreamFactory; ContextMutablePtr updateSettingsForCluster( const Cluster & cluster, ContextPtr context, const Settings & settings, const StorageID & main_table, const SelectQueryInfo * query_info = nullptr, Poco::Logger * log = nullptr); +using AdditionalShardFilterGenerator = std::function; /// Execute a distributed query, creating a query plan, from which the query pipeline can be built. /// `stream_factory` object encapsulates the logic of creating plans for a different type of query /// (currently SELECT, DESCRIBE). @@ -50,7 +51,8 @@ void executeQuery( const ASTPtr & query_ast, ContextPtr context, const SelectQueryInfo & query_info, const ExpressionActionsPtr & sharding_key_expr, const std::string & sharding_key_column_name, - const ClusterPtr & not_optimized_cluster); + const ClusterPtr & not_optimized_cluster, + AdditionalShardFilterGenerator shard_filter_generator = {}); void executeQueryWithParallelReplicas( diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index d1b09707bca..2cfa55f0d87 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -129,13 +129,21 @@ namespace CurrentMetrics { extern const Metric ContextLockWait; extern const Metric BackgroundMovePoolTask; + extern const Metric BackgroundMovePoolSize; extern const Metric BackgroundSchedulePoolTask; + extern const Metric BackgroundSchedulePoolSize; extern const Metric BackgroundBufferFlushSchedulePoolTask; + extern const Metric BackgroundBufferFlushSchedulePoolSize; extern const Metric BackgroundDistributedSchedulePoolTask; + extern const Metric BackgroundDistributedSchedulePoolSize; extern const Metric BackgroundMessageBrokerSchedulePoolTask; + extern const Metric BackgroundMessageBrokerSchedulePoolSize; extern const Metric BackgroundMergesAndMutationsPoolTask; + extern const Metric BackgroundMergesAndMutationsPoolSize; extern const Metric BackgroundFetchesPoolTask; + extern const Metric BackgroundFetchesPoolSize; extern const Metric BackgroundCommonPoolTask; + extern const Metric BackgroundCommonPoolSize; } namespace DB @@ -2175,6 +2183,7 @@ BackgroundSchedulePool & Context::getBufferFlushSchedulePool() const shared->buffer_flush_schedule_pool = std::make_unique( background_buffer_flush_schedule_pool_size, CurrentMetrics::BackgroundBufferFlushSchedulePoolTask, + CurrentMetrics::BackgroundBufferFlushSchedulePoolSize, "BgBufSchPool"); } @@ -2226,6 +2235,7 @@ BackgroundSchedulePool & Context::getSchedulePool() const shared->schedule_pool = std::make_unique( background_schedule_pool_size, CurrentMetrics::BackgroundSchedulePoolTask, + CurrentMetrics::BackgroundSchedulePoolSize, "BgSchPool"); } @@ -2246,6 +2256,7 @@ BackgroundSchedulePool & Context::getDistributedSchedulePool() const shared->distributed_schedule_pool = std::make_unique( background_distributed_schedule_pool_size, CurrentMetrics::BackgroundDistributedSchedulePoolTask, + CurrentMetrics::BackgroundDistributedSchedulePoolSize, "BgDistSchPool"); } @@ -2266,6 +2277,7 @@ BackgroundSchedulePool & Context::getMessageBrokerSchedulePool() const shared->message_broker_schedule_pool = std::make_unique( background_message_broker_schedule_pool_size, CurrentMetrics::BackgroundMessageBrokerSchedulePoolTask, + CurrentMetrics::BackgroundMessageBrokerSchedulePoolSize, "BgMBSchPool"); } @@ -3826,6 +3838,7 @@ void Context::initializeBackgroundExecutorsIfNeeded() /*max_threads_count*/background_pool_size, /*max_tasks_count*/background_pool_size * background_merges_mutations_concurrency_ratio, CurrentMetrics::BackgroundMergesAndMutationsPoolTask, + CurrentMetrics::BackgroundMergesAndMutationsPoolSize, background_merges_mutations_scheduling_policy ); LOG_INFO(shared->log, "Initialized background executor for merges and mutations with num_threads={}, num_tasks={}, scheduling_policy={}", @@ -3836,7 +3849,8 @@ void Context::initializeBackgroundExecutorsIfNeeded() "Move", background_move_pool_size, background_move_pool_size, - CurrentMetrics::BackgroundMovePoolTask + CurrentMetrics::BackgroundMovePoolTask, + CurrentMetrics::BackgroundMovePoolSize ); LOG_INFO(shared->log, "Initialized background executor for move operations with num_threads={}, num_tasks={}", background_move_pool_size, background_move_pool_size); @@ -3845,7 +3859,8 @@ void Context::initializeBackgroundExecutorsIfNeeded() "Fetch", background_fetches_pool_size, background_fetches_pool_size, - CurrentMetrics::BackgroundFetchesPoolTask + CurrentMetrics::BackgroundFetchesPoolTask, + CurrentMetrics::BackgroundFetchesPoolSize ); LOG_INFO(shared->log, "Initialized background executor for fetches with num_threads={}, num_tasks={}", background_fetches_pool_size, background_fetches_pool_size); @@ -3854,7 +3869,8 @@ void Context::initializeBackgroundExecutorsIfNeeded() "Common", background_common_pool_size, background_common_pool_size, - CurrentMetrics::BackgroundCommonPoolTask + CurrentMetrics::BackgroundCommonPoolTask, + CurrentMetrics::BackgroundCommonPoolSize ); LOG_INFO(shared->log, "Initialized background executor for common operations (e.g. clearing old parts) with num_threads={}, num_tasks={}", background_common_pool_size, background_common_pool_size); @@ -4056,21 +4072,34 @@ std::shared_ptr Context::getAsyncReadCounters() const return async_read_counters; } +Context::ParallelReplicasMode Context::getParallelReplicasMode() const +{ + const auto & settings = getSettingsRef(); + + using enum Context::ParallelReplicasMode; + if (!settings.parallel_replicas_custom_key.value.empty()) + return CUSTOM_KEY; + + if (settings.allow_experimental_parallel_reading_from_replicas + && !settings.use_hedged_requests) + return READ_TASKS; + + return SAMPLE_KEY; +} + bool Context::canUseParallelReplicasOnInitiator() const { const auto & settings = getSettingsRef(); - return settings.allow_experimental_parallel_reading_from_replicas + return getParallelReplicasMode() == ParallelReplicasMode::READ_TASKS && settings.max_parallel_replicas > 1 - && !settings.use_hedged_requests && !getClientInfo().collaborate_with_initiator; } bool Context::canUseParallelReplicasOnFollower() const { const auto & settings = getSettingsRef(); - return settings.allow_experimental_parallel_reading_from_replicas + return getParallelReplicasMode() == ParallelReplicasMode::READ_TASKS && settings.max_parallel_replicas > 1 - && !settings.use_hedged_requests && getClientInfo().collaborate_with_initiator; } diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index 19bb6868331..67594a41459 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -1123,6 +1123,15 @@ public: bool canUseParallelReplicasOnInitiator() const; bool canUseParallelReplicasOnFollower() const; + enum class ParallelReplicasMode : uint8_t + { + SAMPLE_KEY, + CUSTOM_KEY, + READ_TASKS, + }; + + ParallelReplicasMode getParallelReplicasMode() const; + private: std::unique_lock getLock() const; diff --git a/src/Interpreters/CrashLog.cpp b/src/Interpreters/CrashLog.cpp index 47a9d62fba6..f1f0ffb6f60 100644 --- a/src/Interpreters/CrashLog.cpp +++ b/src/Interpreters/CrashLog.cpp @@ -80,7 +80,7 @@ void collectCrashLog(Int32 signal, UInt64 thread_id, const String & query_id, co for (size_t i = stack_trace_offset; i < stack_trace_size; ++i) trace.push_back(reinterpret_cast(stack_trace.getFramePointers()[i])); - stack_trace.toStringEveryLine([&trace_full](const std::string & line) { trace_full.push_back(line); }); + stack_trace.toStringEveryLine([&trace_full](std::string_view line) { trace_full.push_back(line); }); CrashLogElement element{static_cast(time / 1000000000), time, signal, thread_id, query_id, trace, trace_full}; crash_log_owned->add(element); diff --git a/src/Interpreters/CrossToInnerJoinVisitor.cpp b/src/Interpreters/CrossToInnerJoinVisitor.cpp index 0675f2bb19c..8755daa1dc8 100644 --- a/src/Interpreters/CrossToInnerJoinVisitor.cpp +++ b/src/Interpreters/CrossToInnerJoinVisitor.cpp @@ -189,7 +189,7 @@ std::vector getTables(const ASTSelectQuery & select) if (t.hasUsing()) { if (has_using) - throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Multuple USING statements are not supported"); + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Multiple USING statements are not supported"); has_using = true; } diff --git a/src/Interpreters/DDLWorker.cpp b/src/Interpreters/DDLWorker.cpp index 7179ce94e0b..f70c0010585 100644 --- a/src/Interpreters/DDLWorker.cpp +++ b/src/Interpreters/DDLWorker.cpp @@ -169,11 +169,11 @@ DDLTaskPtr DDLWorker::initAndCheckTask(const String & entry_name, String & out_r return {}; } - auto write_error_status = [&](const String & host_id, const String & error_message, const String & reason) + auto write_error_status = [&](const String & host_id, const ExecutionStatus & status, const String & reason) { - LOG_ERROR(log, "Cannot parse DDL task {}: {}. Will try to send error status: {}", entry_name, reason, error_message); + LOG_ERROR(log, "Cannot parse DDL task {}: {}. Will try to send error status: {}", entry_name, reason, status.message); createStatusDirs(entry_path, zookeeper); - zookeeper->tryCreate(fs::path(entry_path) / "finished" / host_id, error_message, zkutil::CreateMode::Persistent); + zookeeper->tryCreate(fs::path(entry_path) / "finished" / host_id, status.serializeText(), zkutil::CreateMode::Persistent); }; try @@ -187,7 +187,7 @@ DDLTaskPtr DDLWorker::initAndCheckTask(const String & entry_name, String & out_r /// We can try to create fail node using FQDN if it equal to host name in cluster config attempt will be successful. /// Otherwise, that node will be ignored by DDLQueryStatusSource. out_reason = "Incorrect task format"; - write_error_status(host_fqdn_id, ExecutionStatus::fromCurrentException().serializeText(), out_reason); + write_error_status(host_fqdn_id, ExecutionStatus::fromCurrentException(), out_reason); return {}; } @@ -212,7 +212,7 @@ DDLTaskPtr DDLWorker::initAndCheckTask(const String & entry_name, String & out_r catch (...) { out_reason = "Cannot parse query or obtain cluster info"; - write_error_status(task->host_id_str, ExecutionStatus::fromCurrentException().serializeText(), out_reason); + write_error_status(task->host_id_str, ExecutionStatus::fromCurrentException(), out_reason); return {}; } @@ -542,6 +542,7 @@ void DDLWorker::processTask(DDLTaskBase & task, const ZooKeeperPtr & zookeeper) OpenTelemetry::TracingContextHolder tracing_ctx_holder(__PRETTY_FUNCTION__ , task.entry.tracing_context, this->context->getOpenTelemetrySpanLog()); + tracing_ctx_holder.root_span.kind = OpenTelemetry::CONSUMER; String active_node_path = task.getActiveNodePath(); String finished_node_path = task.getFinishedNodePath(); @@ -650,7 +651,7 @@ void DDLWorker::processTask(DDLTaskBase & task, const ZooKeeperPtr & zookeeper) bool status_written_by_table_or_db = task.ops.empty(); if (status_written_by_table_or_db) { - throw Exception(ErrorCodes::UNFINISHED, "Unexpected error: {}", task.execution_status.serializeText()); + throw Exception(ErrorCodes::UNFINISHED, "Unexpected error: {}", task.execution_status.message); } else { diff --git a/src/Interpreters/DatabaseCatalog.cpp b/src/Interpreters/DatabaseCatalog.cpp index 975e0da66ce..b11a973c7b7 100644 --- a/src/Interpreters/DatabaseCatalog.cpp +++ b/src/Interpreters/DatabaseCatalog.cpp @@ -121,9 +121,16 @@ TemporaryTableHolder::~TemporaryTableHolder() { if (id != UUIDHelpers::Nil) { - auto table = getTable(); - table->flushAndShutdown(); - temporary_tables->dropTable(getContext(), "_tmp_" + toString(id)); + try + { + auto table = getTable(); + table->flushAndShutdown(); + temporary_tables->dropTable(getContext(), "_tmp_" + toString(id)); + } + catch (...) + { + tryLogCurrentException("TemporaryTableHolder"); + } } } @@ -140,7 +147,6 @@ StoragePtr TemporaryTableHolder::getTable() const return table; } - void DatabaseCatalog::initializeAndLoadTemporaryDatabase() { drop_delay_sec = getContext()->getConfigRef().getInt("database_atomic_delay_before_drop_table_sec", default_drop_delay_sec); diff --git a/src/Interpreters/DatabaseCatalog.h b/src/Interpreters/DatabaseCatalog.h index 4200373018d..88645ff72af 100644 --- a/src/Interpreters/DatabaseCatalog.h +++ b/src/Interpreters/DatabaseCatalog.h @@ -235,6 +235,21 @@ public: void checkTableCanBeRemovedOrRenamed(const StorageID & table_id, bool check_referential_dependencies, bool check_loading_dependencies, bool is_drop_database = false) const; + + struct TableMarkedAsDropped + { + StorageID table_id = StorageID::createEmpty(); + StoragePtr table; + String metadata_path; + time_t drop_time{}; + }; + using TablesMarkedAsDropped = std::list; + + TablesMarkedAsDropped getTablesMarkedDropped() + { + std::lock_guard lock(tables_marked_dropped_mutex); + return tables_marked_dropped; + } private: // The global instance of database catalog. unique_ptr is to allow // deferred initialization. Thought I'd use std::optional, but I can't @@ -263,15 +278,6 @@ private: return uuid.toUnderType().items[0] >> (64 - bits_for_first_level); } - struct TableMarkedAsDropped - { - StorageID table_id = StorageID::createEmpty(); - StoragePtr table; - String metadata_path; - time_t drop_time{}; - }; - using TablesMarkedAsDropped = std::list; - void dropTableDataTask(); void dropTableFinally(const TableMarkedAsDropped & table); diff --git a/src/Interpreters/ExpressionActions.cpp b/src/Interpreters/ExpressionActions.cpp index 9931ae97286..4c9f47e5915 100644 --- a/src/Interpreters/ExpressionActions.cpp +++ b/src/Interpreters/ExpressionActions.cpp @@ -848,6 +848,23 @@ std::string ExpressionActions::dumpActions() const return ss.str(); } +void ExpressionActions::describeActions(WriteBuffer & out, std::string_view prefix) const +{ + bool first = true; + + for (const auto & action : actions) + { + out << prefix << (first ? "Actions: " : " "); + out << action.toString() << '\n'; + first = false; + } + + out << prefix << "Positions:"; + for (const auto & pos : result_positions) + out << ' ' << pos; + out << '\n'; +} + JSONBuilder::ItemPtr ExpressionActions::toTree() const { auto inputs_array = std::make_unique(); diff --git a/src/Interpreters/ExpressionActions.h b/src/Interpreters/ExpressionActions.h index 11957997a30..db6670c50b9 100644 --- a/src/Interpreters/ExpressionActions.h +++ b/src/Interpreters/ExpressionActions.h @@ -109,6 +109,9 @@ public: const Block & getSampleBlock() const { return sample_block; } std::string dumpActions() const; + + void describeActions(WriteBuffer & out, std::string_view prefix) const; + JSONBuilder::ItemPtr toTree() const; static NameAndTypePair getSmallestColumn(const NamesAndTypesList & columns); diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index 2b88ff6a353..67aace815dc 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -1089,6 +1089,7 @@ static std::shared_ptr chooseJoinAlgorithm( if (MergeJoin::isSupported(analyzed_join)) return std::make_shared(analyzed_join, right_sample_block); + return std::make_shared(analyzed_join, right_sample_block); } throw Exception(ErrorCodes::NOT_IMPLEMENTED, diff --git a/src/Interpreters/FillingRow.cpp b/src/Interpreters/FillingRow.cpp index b03049a209f..5c2ad548c93 100644 --- a/src/Interpreters/FillingRow.cpp +++ b/src/Interpreters/FillingRow.cpp @@ -107,39 +107,4 @@ void FillingRow::initFromDefaults(size_t from_pos) row[i] = getFillDescription(i).fill_from; } -void insertFromFillingRow(MutableColumns & filling_columns, MutableColumns & interpolate_columns, MutableColumns & other_columns, - const FillingRow & filling_row, const Block & interpolate_block) -{ - for (size_t i = 0, size = filling_columns.size(); i < size; ++i) - { - if (filling_row[i].isNull()) - { - filling_columns[i]->insertDefault(); - } - else - { - filling_columns[i]->insert(filling_row[i]); - } - } - - if (size_t size = interpolate_block.columns()) - { - Columns columns = interpolate_block.getColumns(); - for (size_t i = 0; i < size; ++i) - interpolate_columns[i]->insertFrom(*columns[i]->convertToFullColumnIfConst(), 0); - } - else - for (const auto & interpolate_column : interpolate_columns) - interpolate_column->insertDefault(); - - for (const auto & other_column : other_columns) - other_column->insertDefault(); -} - -void copyRowFromColumns(MutableColumns & dest, const Columns & source, size_t row_num) -{ - for (size_t i = 0, size = source.size(); i < size; ++i) - dest[i]->insertFrom(*source[i], row_num); -} - } diff --git a/src/Interpreters/FillingRow.h b/src/Interpreters/FillingRow.h index 331c237285b..8d47094d0de 100644 --- a/src/Interpreters/FillingRow.h +++ b/src/Interpreters/FillingRow.h @@ -39,8 +39,4 @@ private: SortDescription sort_description; }; -void insertFromFillingRow(MutableColumns & filling_columns, MutableColumns & interpolate_columns, MutableColumns & other_columns, - const FillingRow & filling_row, const Block & interpolate_block); -void copyRowFromColumns(MutableColumns & dest, const Columns & source, size_t row_num); - } diff --git a/src/Interpreters/FullSortingMergeJoin.h b/src/Interpreters/FullSortingMergeJoin.h index fa7d0478535..a94d7a7dfc6 100644 --- a/src/Interpreters/FullSortingMergeJoin.h +++ b/src/Interpreters/FullSortingMergeJoin.h @@ -44,6 +44,10 @@ public: const auto & on_expr = table_join->getOnlyClause(); bool support_conditions = !on_expr.on_filter_condition_left && !on_expr.on_filter_condition_right; + if (!on_expr.analyzer_left_filter_condition_column_name.empty() || + !on_expr.analyzer_right_filter_condition_column_name.empty()) + support_conditions = false; + /// Key column can change nullability and it's not handled on type conversion stage, so algorithm should be aware of it bool support_using_and_nulls = !table_join->hasUsing() || !table_join->joinUseNulls(); diff --git a/src/Interpreters/GraceHashJoin.cpp b/src/Interpreters/GraceHashJoin.cpp index 79a825a752f..7795061072c 100644 --- a/src/Interpreters/GraceHashJoin.cpp +++ b/src/Interpreters/GraceHashJoin.cpp @@ -10,7 +10,6 @@ #include #include #include -#include #include #include diff --git a/src/Interpreters/GraceHashJoin.h b/src/Interpreters/GraceHashJoin.h index 4f7694e2f07..b8d83f4cad0 100644 --- a/src/Interpreters/GraceHashJoin.h +++ b/src/Interpreters/GraceHashJoin.h @@ -139,6 +139,7 @@ private: mutable SharedMutex rehash_mutex; FileBucket * current_bucket = nullptr; + mutable std::mutex current_bucket_mutex; InMemoryJoinPtr hash_join; diff --git a/src/Interpreters/HashJoin.cpp b/src/Interpreters/HashJoin.cpp index fba985da41c..b4376426700 100644 --- a/src/Interpreters/HashJoin.cpp +++ b/src/Interpreters/HashJoin.cpp @@ -495,7 +495,7 @@ size_t HashJoin::getTotalByteCount() const if (!data) return 0; -#ifdef NDEBUG +#ifndef NDEBUG size_t debug_blocks_allocated_size = 0; for (const auto & block : data->blocks) debug_blocks_allocated_size += block.allocatedBytes(); diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index 29b7a4db609..7a4d65a4d57 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -9,7 +9,7 @@ #include #include #include -#include +#include #include #include @@ -940,23 +940,32 @@ void InterpreterCreateQuery::setEngine(ASTCreateQuery & create) const if (create.temporary) { - if (create.storage && create.storage->engine && create.storage->engine->name != "Memory") - throw Exception(ErrorCodes::INCORRECT_QUERY, "Temporary tables can only be created with ENGINE = Memory, not {}", - create.storage->engine->name); - /// It's possible if some part of storage definition (such as PARTITION BY) is specified, but ENGINE is not. /// It makes sense when default_table_engine setting is used, but not for temporary tables. /// For temporary tables we ignore this setting to allow CREATE TEMPORARY TABLE query without specifying ENGINE - /// even if setting is set to MergeTree or something like that (otherwise MergeTree will be substituted and query will fail). - if (create.storage && !create.storage->engine) - throw Exception(ErrorCodes::INCORRECT_QUERY, "Invalid storage definition for temporary table: must be either ENGINE = Memory or empty"); - auto engine_ast = std::make_shared(); - engine_ast->name = "Memory"; - engine_ast->no_empty_args = true; - auto storage_ast = std::make_shared(); - storage_ast->set(storage_ast->engine, engine_ast); - create.set(create.storage, storage_ast); + if (!create.cluster.empty()) + throw Exception(ErrorCodes::INCORRECT_QUERY, "Temporary tables cannot be created with ON CLUSTER clause"); + + if (create.storage) + { + if (create.storage->engine) + { + if (create.storage->engine->name.starts_with("Replicated") || create.storage->engine->name == "KeeperMap") + throw Exception(ErrorCodes::INCORRECT_QUERY, "Temporary tables cannot be created with Replicated or KeeperMap table engines"); + } + else + throw Exception(ErrorCodes::INCORRECT_QUERY, "Invalid storage definition for temporary table"); + } + else + { + auto engine_ast = std::make_shared(); + engine_ast->name = "Memory"; + engine_ast->no_empty_args = true; + auto storage_ast = std::make_shared(); + storage_ast->set(storage_ast->engine, engine_ast); + create.set(create.storage, storage_ast); + } return; } @@ -1284,8 +1293,21 @@ bool InterpreterCreateQuery::doCreateTable(ASTCreateQuery & create, if (create.if_not_exists && getContext()->tryResolveStorageID({"", create.getTable()}, Context::ResolveExternal)) return false; + DatabasePtr database = DatabaseCatalog::instance().getDatabase(DatabaseCatalog::TEMPORARY_DATABASE); + String temporary_table_name = create.getTable(); - auto temporary_table = TemporaryTableHolder(getContext(), properties.columns, properties.constraints, query_ptr); + auto creator = [&](const StorageID & table_id) + { + return StorageFactory::instance().get(create, + database->getTableDataPath(table_id.getTableName()), + getContext(), + getContext()->getGlobalContext(), + properties.columns, + properties.constraints, + false); + }; + auto temporary_table = TemporaryTableHolder(getContext(), creator, query_ptr); + getContext()->getSessionContext()->addExternalTable(temporary_table_name, std::move(temporary_table)); return true; } @@ -1712,7 +1734,13 @@ AccessRightsElements InterpreterCreateQuery::getRequiredAccess() const else { if (create.temporary) - required_access.emplace_back(AccessType::CREATE_TEMPORARY_TABLE); + { + /// Currently default table engine for temporary tables is Memory. default_table_engine does not affect temporary tables. + if (create.storage && create.storage->engine && create.storage->engine->name != "Memory") + required_access.emplace_back(AccessType::CREATE_ARBITRARY_TEMPORARY_TABLE); + else + required_access.emplace_back(AccessType::CREATE_TEMPORARY_TABLE); + } else { if (create.replace_table) diff --git a/src/Interpreters/InterpreterDropQuery.cpp b/src/Interpreters/InterpreterDropQuery.cpp index f4507de5ac7..e16403bed67 100644 --- a/src/Interpreters/InterpreterDropQuery.cpp +++ b/src/Interpreters/InterpreterDropQuery.cpp @@ -282,11 +282,6 @@ BlockIO InterpreterDropQuery::executeToTemporaryTable(const String & table_name, else if (kind == ASTDropQuery::Kind::Drop) { context_handle->removeExternalTable(table_name); - table->flushAndShutdown(); - auto table_lock = table->lockExclusively(getContext()->getCurrentQueryId(), getContext()->getSettingsRef().lock_acquire_timeout); - /// Delete table data - table->drop(); - table->is_dropped = true; } else if (kind == ASTDropQuery::Kind::Detach) { diff --git a/src/Interpreters/InterpreterKillQueryQuery.cpp b/src/Interpreters/InterpreterKillQueryQuery.cpp index 40698386ccb..3330159aff5 100644 --- a/src/Interpreters/InterpreterKillQueryQuery.cpp +++ b/src/Interpreters/InterpreterKillQueryQuery.cpp @@ -161,6 +161,8 @@ public: if (curr_process.processed) continue; + LOG_DEBUG(&Poco::Logger::get("KillQuery"), "Will kill query {} (synchronously)", curr_process.query_id); + auto code = process_list.sendCancelToQuery(curr_process.query_id, curr_process.user, true); if (code != CancellationCode::QueryIsNotInitializedYet && code != CancellationCode::CancelSent) @@ -226,6 +228,8 @@ BlockIO InterpreterKillQueryQuery::execute() MutableColumns res_columns = header.cloneEmptyColumns(); for (const auto & query_desc : queries_to_stop) { + if (!query.test) + LOG_DEBUG(&Poco::Logger::get("KillQuery"), "Will kill query {} (asynchronously)", query_desc.query_id); auto code = (query.test) ? CancellationCode::Unknown : process_list.sendCancelToQuery(query_desc.query_id, query_desc.user, true); insertResultRow(query_desc.source_num, code, processes_block, header, res_columns); } diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 318ea5fdf42..79b073e30f3 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -38,6 +38,7 @@ #include #include #include +#include #include #include @@ -114,6 +115,7 @@ namespace ErrorCodes extern const int INVALID_WITH_FILL_EXPRESSION; extern const int ACCESS_DENIED; extern const int UNKNOWN_IDENTIFIER; + extern const int BAD_ARGUMENTS; } /// Assumes `storage` is set and the table filter (row-level security) is not empty. @@ -229,10 +231,13 @@ InterpreterSelectQuery::InterpreterSelectQuery( InterpreterSelectQuery::~InterpreterSelectQuery() = default; +namespace +{ + /** There are no limits on the maximum size of the result for the subquery. * Since the result of the query is not the result of the entire query. */ -static ContextPtr getSubqueryContext(const ContextPtr & context) +ContextPtr getSubqueryContext(const ContextPtr & context) { auto subquery_context = Context::createCopy(context); Settings subquery_settings = context->getSettings(); @@ -244,7 +249,7 @@ static ContextPtr getSubqueryContext(const ContextPtr & context) return subquery_context; } -static void rewriteMultipleJoins(ASTPtr & query, const TablesWithColumns & tables, const String & database, const Settings & settings) +void rewriteMultipleJoins(ASTPtr & query, const TablesWithColumns & tables, const String & database, const Settings & settings) { ASTSelectQuery & select = query->as(); @@ -264,7 +269,7 @@ static void rewriteMultipleJoins(ASTPtr & query, const TablesWithColumns & table } /// Checks that the current user has the SELECT privilege. -static void checkAccessRightsForSelect( +void checkAccessRightsForSelect( const ContextPtr & context, const StorageID & table_id, const StorageMetadataPtr & table_metadata, @@ -294,7 +299,7 @@ static void checkAccessRightsForSelect( context->checkAccess(AccessType::SELECT, table_id, syntax_analyzer_result.requiredSourceColumnsForAccessCheck()); } -static ASTPtr parseAdditionalFilterConditionForTable( +ASTPtr parseAdditionalFilterConditionForTable( const Map & setting, const DatabaseAndTableWithAlias & target, const Context & context) @@ -322,7 +327,7 @@ static ASTPtr parseAdditionalFilterConditionForTable( } /// Returns true if we should ignore quotas and limits for a specified table in the system database. -static bool shouldIgnoreQuotaAndLimits(const StorageID & table_id) +bool shouldIgnoreQuotaAndLimits(const StorageID & table_id) { if (table_id.database_name == DatabaseCatalog::SYSTEM_DATABASE) { @@ -333,6 +338,8 @@ static bool shouldIgnoreQuotaAndLimits(const StorageID & table_id) return false; } +} + InterpreterSelectQuery::InterpreterSelectQuery( const ASTPtr & query_ptr_, const ContextPtr & context_, @@ -448,10 +455,11 @@ InterpreterSelectQuery::InterpreterSelectQuery( } } - if (joined_tables.tablesCount() > 1 && settings.allow_experimental_parallel_reading_from_replicas) + if (joined_tables.tablesCount() > 1 && (!settings.parallel_replicas_custom_key.value.empty() || settings.allow_experimental_parallel_reading_from_replicas)) { LOG_WARNING(log, "Joins are not supported with parallel replicas. Query will be executed without using them."); context->setSetting("allow_experimental_parallel_reading_from_replicas", false); + context->setSetting("parallel_replicas_custom_key", String{""}); } /// Rewrite JOINs @@ -509,6 +517,42 @@ InterpreterSelectQuery::InterpreterSelectQuery( query_info.additional_filter_ast = parseAdditionalFilterConditionForTable( settings.additional_table_filters, joined_tables.tablesWithColumns().front().table, *context); + ASTPtr parallel_replicas_custom_filter_ast = nullptr; + if (context->getParallelReplicasMode() == Context::ParallelReplicasMode::CUSTOM_KEY && !joined_tables.tablesWithColumns().empty()) + { + if (settings.parallel_replicas_count > 1) + { + if (auto custom_key_ast = parseCustomKeyForTable(settings.parallel_replicas_custom_key, *context)) + { + LOG_TRACE(log, "Processing query on a replica using custom_key '{}'", settings.parallel_replicas_custom_key.value); + if (!storage) + throw DB::Exception(ErrorCodes::BAD_ARGUMENTS, "Storage is unknown when trying to parse custom key for parallel replica"); + + parallel_replicas_custom_filter_ast = getCustomKeyFilterForParallelReplica( + settings.parallel_replicas_count, + settings.parallel_replica_offset, + std::move(custom_key_ast), + settings.parallel_replicas_custom_key_filter_type, + *storage, + context); + } + else if (settings.parallel_replica_offset > 0) + { + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Parallel replicas processing with custom_key has been requested " + "(setting 'max_parallel_replicas') but the table does not have custom_key defined for it " + "or it's invalid (settings `parallel_replicas_custom_key`)"); + } + } + else if (auto * distributed = dynamic_cast(storage.get()); + distributed && canUseCustomKey(settings, *distributed->getCluster(), *context)) + { + query_info.use_custom_key = true; + context->setSetting("distributed_group_by_no_merge", 2); + } + } + if (autoFinalOnQuery(query)) { query.setFinal(); @@ -518,7 +562,6 @@ InterpreterSelectQuery::InterpreterSelectQuery( { /// Allow push down and other optimizations for VIEW: replace with subquery and rewrite it. ASTPtr view_table; - NameToNameMap parameter_values; NameToNameMap parameter_types; if (view) { @@ -531,14 +574,13 @@ InterpreterSelectQuery::InterpreterSelectQuery( /// and after query is replaced, we use these parameters to substitute in the parameterized view query if (query_info.is_parameterized_view) { - parameter_values = analyzeFunctionParamValues(query_ptr); - view->setParameterValues(parameter_values); - parameter_types = view->getParameterValues(); + query_info.parameterized_view_values = analyzeFunctionParamValues(query_ptr); + parameter_types = view->getParameterTypes(); } view->replaceWithSubquery(getSelectQuery(), view_table, metadata_snapshot, view->isParameterizedView()); if (query_info.is_parameterized_view) { - view->replaceQueryParametersIfParametrizedView(query_ptr); + view->replaceQueryParametersIfParametrizedView(query_ptr, query_info.parameterized_view_values); } } @@ -551,7 +593,7 @@ InterpreterSelectQuery::InterpreterSelectQuery( required_result_column_names, table_join, query_info.is_parameterized_view, - parameter_values, + query_info.parameterized_view_values, parameter_types); @@ -593,8 +635,6 @@ InterpreterSelectQuery::InterpreterSelectQuery( Names queried_columns = syntax_analyzer_result->requiredSourceColumns(); const auto & supported_prewhere_columns = storage->supportedPrewhereColumns(); - if (supported_prewhere_columns.has_value()) - std::erase_if(queried_columns, [&](const auto & name) { return !supported_prewhere_columns->contains(name); }); MergeTreeWhereOptimizer{ current_info, @@ -602,6 +642,7 @@ InterpreterSelectQuery::InterpreterSelectQuery( std::move(column_compressed_sizes), metadata_snapshot, queried_columns, + supported_prewhere_columns, log}; } } @@ -694,7 +735,17 @@ InterpreterSelectQuery::InterpreterSelectQuery( query_info.filter_asts.push_back(query_info.additional_filter_ast); } - source_header = storage_snapshot->getSampleBlockForColumns(required_columns, parameter_values); + if (parallel_replicas_custom_filter_ast) + { + parallel_replicas_custom_filter_info = generateFilterActions( + table_id, parallel_replicas_custom_filter_ast, context, storage, storage_snapshot, metadata_snapshot, required_columns, + prepared_sets); + + parallel_replicas_custom_filter_info->do_remove_column = true; + query_info.filter_asts.push_back(parallel_replicas_custom_filter_ast); + } + + source_header = storage_snapshot->getSampleBlockForColumns(required_columns, query_info.parameterized_view_values); } /// Calculate structure of the result. @@ -1436,17 +1487,23 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, std::optional

( + auto filter_step = std::make_unique( query_plan.getCurrentDataStream(), - additional_filter_info->actions, - additional_filter_info->column_name, - additional_filter_info->do_remove_column); + new_filter_info->actions, + new_filter_info->column_name, + new_filter_info->do_remove_column); - additional_filter_step->setStepDescription("Additional filter"); - query_plan.addStep(std::move(additional_filter_step)); - } + filter_step->setStepDescription(description); + query_plan.addStep(std::move(filter_step)); + }; + + if (additional_filter_info) + add_filter_step(additional_filter_info, "Additional filter"); + + if (parallel_replicas_custom_filter_info) + add_filter_step(parallel_replicas_custom_filter_info, "Parallel replica custom key filter"); if (expressions.before_array_join) { diff --git a/src/Interpreters/InterpreterSelectQuery.h b/src/Interpreters/InterpreterSelectQuery.h index 140e10a5f81..58fddb8ffe9 100644 --- a/src/Interpreters/InterpreterSelectQuery.h +++ b/src/Interpreters/InterpreterSelectQuery.h @@ -215,6 +215,9 @@ private: /// For additional_filter setting. FilterDAGInfoPtr additional_filter_info; + /// For "per replica" filter when multiple replicas are used + FilterDAGInfoPtr parallel_replicas_custom_filter_info; + QueryProcessingStage::Enum from_stage = QueryProcessingStage::FetchColumns; /// List of columns to read to execute the query. diff --git a/src/Interpreters/InterpreterSelectQueryAnalyzer.cpp b/src/Interpreters/InterpreterSelectQueryAnalyzer.cpp index 0536ee10f7c..98f70c25dcd 100644 --- a/src/Interpreters/InterpreterSelectQueryAnalyzer.cpp +++ b/src/Interpreters/InterpreterSelectQueryAnalyzer.cpp @@ -226,6 +226,12 @@ BlockIO InterpreterSelectQueryAnalyzer::execute() return result; } +QueryPlan & InterpreterSelectQueryAnalyzer::getQueryPlan() +{ + planner.buildQueryPlanIfNeeded(); + return planner.getQueryPlan(); +} + QueryPlan && InterpreterSelectQueryAnalyzer::extractQueryPlan() && { planner.buildQueryPlanIfNeeded(); diff --git a/src/Interpreters/InterpreterSelectQueryAnalyzer.h b/src/Interpreters/InterpreterSelectQueryAnalyzer.h index 681a9cfe5a3..2c8af49cf0e 100644 --- a/src/Interpreters/InterpreterSelectQueryAnalyzer.h +++ b/src/Interpreters/InterpreterSelectQueryAnalyzer.h @@ -51,6 +51,8 @@ public: BlockIO execute() override; + QueryPlan & getQueryPlan(); + QueryPlan && extractQueryPlan() &&; QueryPipelineBuilder buildQueryPipeline(); diff --git a/src/Interpreters/InterpreterSetQuery.cpp b/src/Interpreters/InterpreterSetQuery.cpp index 2c8611ffb63..3c5a154c572 100644 --- a/src/Interpreters/InterpreterSetQuery.cpp +++ b/src/Interpreters/InterpreterSetQuery.cpp @@ -1,6 +1,11 @@ #include #include #include +#include +#include +#include +#include +#include namespace DB { @@ -26,4 +31,59 @@ void InterpreterSetQuery::executeForCurrentContext() getContext()->resetSettingsToDefaultValue(ast.default_settings); } +static void applySettingsFromSelectWithUnion(const ASTSelectWithUnionQuery & select_with_union, ContextMutablePtr context) +{ + const ASTs & children = select_with_union.list_of_selects->children; + if (children.empty()) + return; + + // We might have an arbitrarily complex UNION tree, so just give + // up if the last first-order child is not a plain SELECT. + // It is flattened later, when we process UNION ALL/DISTINCT. + const auto * last_select = children.back()->as(); + if (last_select && last_select->settings()) + { + InterpreterSetQuery(last_select->settings(), context).executeForCurrentContext(); + } +} + +void InterpreterSetQuery::applySettingsFromQuery(const ASTPtr & ast, ContextMutablePtr context_) +{ + if (!ast) + return; + + if (const auto * select_query = ast->as()) + { + if (auto new_settings = select_query->settings()) + InterpreterSetQuery(new_settings, context_).executeForCurrentContext(); + } + else if (const auto * select_with_union_query = ast->as()) + { + applySettingsFromSelectWithUnion(*select_with_union_query, context_); + } + else if (const auto * explain_query = ast->as()) + { + applySettingsFromQuery(explain_query->getExplainedQuery(), context_); + } + else if (const auto * query_with_output = dynamic_cast(ast.get())) + { + if (query_with_output->settings_ast) + InterpreterSetQuery(query_with_output->settings_ast, context_).executeForCurrentContext(); + + if (const auto * create_query = ast->as()) + { + if (create_query->select) + { + applySettingsFromSelectWithUnion(create_query->select->as(), context_); + } + } + } + else if (auto * insert_query = ast->as()) + { + context_->setInsertFormat(insert_query->format); + if (insert_query->settings_ast) + InterpreterSetQuery(insert_query->settings_ast, context_).executeForCurrentContext(); + } +} + } diff --git a/src/Interpreters/InterpreterSetQuery.h b/src/Interpreters/InterpreterSetQuery.h index 39d331100d6..bcd4022f9bb 100644 --- a/src/Interpreters/InterpreterSetQuery.h +++ b/src/Interpreters/InterpreterSetQuery.h @@ -27,6 +27,9 @@ public: bool supportsTransactions() const override { return true; } + /// To apply SETTINGS clauses from query as early as possible + static void applySettingsFromQuery(const ASTPtr & ast, ContextMutablePtr context_); + private: ASTPtr query_ptr; }; diff --git a/src/Interpreters/LogicalExpressionsOptimizer.cpp b/src/Interpreters/LogicalExpressionsOptimizer.cpp index 02594269f08..5dd1b510480 100644 --- a/src/Interpreters/LogicalExpressionsOptimizer.cpp +++ b/src/Interpreters/LogicalExpressionsOptimizer.cpp @@ -119,7 +119,9 @@ void LogicalExpressionsOptimizer::collectDisjunctiveEqualityChains() bool found_chain = false; auto * function = to_node->as(); - if (function && function->name == "or" && function->children.size() == 1) + /// Optimization does not respect aliases properly, which can lead to MULTIPLE_EXPRESSION_FOR_ALIAS error. + /// Disable it if an expression has an alias. Proper implementation is done with the new analyzer. + if (function && function->alias.empty() && function->name == "or" && function->children.size() == 1) { const auto * expression_list = function->children[0]->as(); if (expression_list) @@ -128,14 +130,14 @@ void LogicalExpressionsOptimizer::collectDisjunctiveEqualityChains() for (const auto & child : expression_list->children) { auto * equals = child->as(); - if (equals && equals->name == "equals" && equals->children.size() == 1) + if (equals && equals->alias.empty() && equals->name == "equals" && equals->children.size() == 1) { const auto * equals_expression_list = equals->children[0]->as(); if (equals_expression_list && equals_expression_list->children.size() == 2) { /// Equality expr = xN. const auto * literal = equals_expression_list->children[1]->as(); - if (literal) + if (literal && literal->alias.empty()) { auto expr_lhs = equals_expression_list->children[0]->getTreeHash(); OrWithExpression or_with_expression{function, expr_lhs, function->tryGetAlias()}; @@ -230,6 +232,9 @@ bool LogicalExpressionsOptimizer::mayOptimizeDisjunctiveEqualityChain(const Disj const auto & equalities = chain.second; const auto & equality_functions = equalities.functions; + if (settings.optimize_min_equality_disjunction_chain_length == 0) + return false; + /// For LowCardinality column, the dict is usually smaller and the index is relatively large. /// In most cases, merging OR-chain as IN is better than converting each LowCardinality into full column individually. /// For non-LowCardinality, we need to eliminate too short chains. diff --git a/src/Interpreters/MergeTreeTransaction.cpp b/src/Interpreters/MergeTreeTransaction.cpp index f16ece46530..bfdda354c9b 100644 --- a/src/Interpreters/MergeTreeTransaction.cpp +++ b/src/Interpreters/MergeTreeTransaction.cpp @@ -168,6 +168,8 @@ void MergeTreeTransaction::addMutation(const StoragePtr & table, const String & bool MergeTreeTransaction::isReadOnly() const { std::lock_guard lock{mutex}; + if (finalized) + return is_read_only; chassert((creating_parts.empty() && removing_parts.empty() && mutations.empty()) == storages.empty()); return storages.empty(); } @@ -315,6 +317,20 @@ bool MergeTreeTransaction::rollback() noexcept return true; } +void MergeTreeTransaction::afterFinalize() +{ + std::lock_guard lock{mutex}; + chassert((creating_parts.empty() && removing_parts.empty() && mutations.empty()) == storages.empty()); + + /// Remember if it was read-only transaction before we clear storages + is_read_only = storages.empty(); + + /// Release shared pointers just in case + storages.clear(); + mutations.clear(); + finalized = true; +} + void MergeTreeTransaction::onException() { TransactionLog::instance().rollbackTransaction(shared_from_this()); @@ -331,6 +347,11 @@ String MergeTreeTransaction::dumpDescription() const } std::lock_guard lock{mutex}; + if (finalized) + { + res += ", cannot dump detailed description, transaction is finalized"; + return res; + } res += fmt::format(", affects {} tables:", storages.size()); diff --git a/src/Interpreters/MergeTreeTransaction.h b/src/Interpreters/MergeTreeTransaction.h index f2d8d29d244..4ca36cf64ad 100644 --- a/src/Interpreters/MergeTreeTransaction.h +++ b/src/Interpreters/MergeTreeTransaction.h @@ -65,6 +65,8 @@ private: scope_guard beforeCommit(); void afterCommit(CSN assigned_csn) noexcept; bool rollback() noexcept; + void afterFinalize(); + void checkIsNotCancelled() const; mutable std::mutex mutex; @@ -74,6 +76,11 @@ private: std::atomic snapshot; const std::list::iterator snapshot_in_use_it; + bool finalized TSA_GUARDED_BY(mutex) = false; + + /// Indicates if transaction was read-only before `afterFinalize` + bool is_read_only TSA_GUARDED_BY(mutex) = false; + /// Lists of changes made by transaction std::unordered_set storages TSA_GUARDED_BY(mutex); DataPartsVector creating_parts TSA_GUARDED_BY(mutex); diff --git a/src/Interpreters/OpenTelemetrySpanLog.cpp b/src/Interpreters/OpenTelemetrySpanLog.cpp index 57d5c11ad97..257c64e1df8 100644 --- a/src/Interpreters/OpenTelemetrySpanLog.cpp +++ b/src/Interpreters/OpenTelemetrySpanLog.cpp @@ -8,9 +8,10 @@ #include #include #include +#include #include -#include +#include #include #include @@ -20,11 +21,23 @@ namespace DB NamesAndTypesList OpenTelemetrySpanLogElement::getNamesAndTypes() { + auto span_kind_type = std::make_shared( + DataTypeEnum8::Values + { + {"INTERNAL", static_cast(OpenTelemetry::INTERNAL)}, + {"SERVER", static_cast(OpenTelemetry::SERVER)}, + {"CLIENT", static_cast(OpenTelemetry::CLIENT)}, + {"PRODUCER", static_cast(OpenTelemetry::PRODUCER)}, + {"CONSUMER", static_cast(OpenTelemetry::CONSUMER)} + } + ); + return { {"trace_id", std::make_shared()}, {"span_id", std::make_shared()}, {"parent_span_id", std::make_shared()}, {"operation_name", std::make_shared()}, + {"kind", std::move(span_kind_type)}, // DateTime64 is really unwieldy -- there is no "normal" way to convert // it to an UInt64 count of microseconds, except: // 1) reinterpretAsUInt64(reinterpretAsFixedString(date)), which just @@ -59,6 +72,7 @@ void OpenTelemetrySpanLogElement::appendToBlock(MutableColumns & columns) const columns[i++]->insert(span_id); columns[i++]->insert(parent_span_id); columns[i++]->insert(operation_name); + columns[i++]->insert(kind); columns[i++]->insert(start_time_us); columns[i++]->insert(finish_time_us); columns[i++]->insert(DateLUT::instance().toDayNum(finish_time_us / 1000000).toUnderType()); diff --git a/src/Interpreters/OptimizeIfChains.cpp b/src/Interpreters/OptimizeIfChains.cpp index ba4c7bcd95f..9a5f9bcb2e1 100644 --- a/src/Interpreters/OptimizeIfChains.cpp +++ b/src/Interpreters/OptimizeIfChains.cpp @@ -64,6 +64,7 @@ ASTs OptimizeIfChainsVisitor::ifChain(const ASTPtr & child) throw Exception(ErrorCodes::UNEXPECTED_AST_STRUCTURE, "Unexpected AST for function 'if'"); const auto * function_args = function_node->arguments->as(); + chassert(function_args); if (!function_args || function_args->children.size() != 3) throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, diff --git a/src/Interpreters/ProcessList.cpp b/src/Interpreters/ProcessList.cpp index 49d7989ac5e..b792ea538ae 100644 --- a/src/Interpreters/ProcessList.cpp +++ b/src/Interpreters/ProcessList.cpp @@ -362,7 +362,11 @@ QueryStatus::QueryStatus( QueryStatus::~QueryStatus() { - assert(executors.empty()); +#if !defined(NDEBUG) + /// Check that all executors were invalidated. + for (const auto & e : executors) + assert(!e->executor); +#endif if (auto * memory_tracker = getMemoryTracker()) { @@ -373,6 +377,19 @@ QueryStatus::~QueryStatus() } } +void QueryStatus::ExecutorHolder::cancel() +{ + std::lock_guard lock(mutex); + if (executor) + executor->cancel(); +} + +void QueryStatus::ExecutorHolder::remove() +{ + std::lock_guard lock(mutex); + executor = nullptr; +} + CancellationCode QueryStatus::cancelQuery(bool) { if (is_killed.load()) @@ -380,8 +397,25 @@ CancellationCode QueryStatus::cancelQuery(bool) is_killed.store(true); - std::lock_guard lock(executors_mutex); - for (auto * e : executors) + std::vector executors_snapshot; + + { + /// Create a snapshot of executors under a mutex. + std::lock_guard lock(executors_mutex); + executors_snapshot = executors; + } + + /// We should call cancel() for each executor with unlocked executors_mutex, because + /// cancel() can try to lock some internal mutex that is already locked by query executing + /// thread, and query executing thread can call removePipelineExecutor and lock executors_mutex, + /// which will lead to deadlock. + /// Note that the size and the content of executors cannot be changed while + /// executors_mutex is unlocked, because: + /// 1) We don't allow adding new executors while cancelling query in addPipelineExecutor + /// 2) We don't actually remove executor holder from executors in removePipelineExecutor, + /// just mark that executor is invalid. + /// So, it's ok to use a snapshot created above under a mutex, it won't be any differ from actual executors. + for (const auto & e : executors_snapshot) e->cancel(); return CancellationCode::CancelSent; @@ -396,15 +430,17 @@ void QueryStatus::addPipelineExecutor(PipelineExecutor * e) throw Exception(ErrorCodes::QUERY_WAS_CANCELLED, "Query was cancelled"); std::lock_guard lock(executors_mutex); - assert(std::find(executors.begin(), executors.end(), e) == executors.end()); - executors.push_back(e); + assert(std::find_if(executors.begin(), executors.end(), [e](const ExecutorHolderPtr & x){ return x->executor == e; }) == executors.end()); + executors.push_back(std::make_shared(e)); } void QueryStatus::removePipelineExecutor(PipelineExecutor * e) { std::lock_guard lock(executors_mutex); - assert(std::find(executors.begin(), executors.end(), e) != executors.end()); - std::erase_if(executors, [e](PipelineExecutor * x) { return x == e; }); + auto it = std::find_if(executors.begin(), executors.end(), [e](const ExecutorHolderPtr & x){ return x->executor == e; }); + assert(it != executors.end()); + /// Invalidate executor pointer inside holder, but don't remove holder from the executors (to avoid race with cancelQuery) + (*it)->remove(); } bool QueryStatus::checkTimeLimit() diff --git a/src/Interpreters/ProcessList.h b/src/Interpreters/ProcessList.h index d5c136ab62a..30bfde4e218 100644 --- a/src/Interpreters/ProcessList.h +++ b/src/Interpreters/ProcessList.h @@ -119,8 +119,22 @@ protected: mutable std::mutex executors_mutex; + struct ExecutorHolder + { + ExecutorHolder(PipelineExecutor * e) : executor(e) {} + + void cancel(); + + void remove(); + + PipelineExecutor * executor; + std::mutex mutex; + }; + + using ExecutorHolderPtr = std::shared_ptr; + /// Array of PipelineExecutors to be cancelled when a cancelQuery is received - std::vector executors; + std::vector executors; enum QueryStreamsStatus { diff --git a/src/Interpreters/ReplaceQueryParameterVisitor.cpp b/src/Interpreters/ReplaceQueryParameterVisitor.cpp index f271de26ca4..fad9d4bbfb2 100644 --- a/src/Interpreters/ReplaceQueryParameterVisitor.cpp +++ b/src/Interpreters/ReplaceQueryParameterVisitor.cpp @@ -50,7 +50,16 @@ void ReplaceQueryParameterVisitor::visit(ASTPtr & ast) void ReplaceQueryParameterVisitor::visitChildren(ASTPtr & ast) { for (auto & child : ast->children) + { + void * old_ptr = child.get(); visit(child); + void * new_ptr = child.get(); + + /// Some AST classes have naked pointers to children elements as members. + /// We have to replace them if the child was replaced. + if (new_ptr != old_ptr) + ast->updatePointerToChild(old_ptr, new_ptr); + } } const String & ReplaceQueryParameterVisitor::getParamValue(const String & name) @@ -74,7 +83,10 @@ void ReplaceQueryParameterVisitor::visitQueryParameter(ASTPtr & ast) IColumn & temp_column = *temp_column_ptr; ReadBufferFromString read_buffer{value}; FormatSettings format_settings; - data_type->getDefaultSerialization()->deserializeTextEscaped(temp_column, read_buffer, format_settings); + if (ast_param.name == "_request_body") + data_type->getDefaultSerialization()->deserializeWholeText(temp_column, read_buffer, format_settings); + else + data_type->getDefaultSerialization()->deserializeTextEscaped(temp_column, read_buffer, format_settings); if (!read_buffer.eof()) throw Exception(ErrorCodes::BAD_QUERY_PARAMETER, @@ -89,6 +101,7 @@ void ReplaceQueryParameterVisitor::visitQueryParameter(ASTPtr & ast) literal = value; else literal = temp_column[0]; + ast = addTypeConversionToAST(std::make_shared(literal), type_name); /// Keep the original alias. diff --git a/src/Interpreters/Session.cpp b/src/Interpreters/Session.cpp index 7411050aa2d..70d4c0e6ae0 100644 --- a/src/Interpreters/Session.cpp +++ b/src/Interpreters/Session.cpp @@ -140,6 +140,23 @@ public: scheduleCloseSession(session, lock); } + void closeSession(const UUID & user_id, const String & session_id) + { + std::unique_lock lock(mutex); + Key key{user_id, session_id}; + auto it = sessions.find(key); + if (it == sessions.end()) + { + LOG_INFO(log, "Session {} not found for user {}, probably it's already closed", session_id, user_id); + return; + } + + if (!it->second.unique()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot close session {} with refcount {}", session_id, it->second.use_count()); + + sessions.erase(it); + } + private: class SessionKeyHash { @@ -408,7 +425,7 @@ ContextMutablePtr Session::makeSessionContext(const String & session_name_, std: std::shared_ptr new_named_session; bool new_named_session_created = false; std::tie(new_named_session, new_named_session_created) - = NamedSessionsStorage::instance().acquireSession(global_context, user_id.value_or(UUID{}), session_name_, timeout_, session_check_); + = NamedSessionsStorage::instance().acquireSession(global_context, *user_id, session_name_, timeout_, session_check_); auto new_session_context = new_named_session->context; new_session_context->makeSessionContext(); @@ -533,5 +550,18 @@ void Session::releaseSessionID() named_session = nullptr; } +void Session::closeSession(const String & session_id) +{ + if (!user_id) /// User was not authenticated + return; + + /// named_session may be not set due to an early exception + if (!named_session) + return; + + releaseSessionID(); + NamedSessionsStorage::instance().closeSession(*user_id, session_id); +} + } diff --git a/src/Interpreters/Session.h b/src/Interpreters/Session.h index 0f17c378915..443867806d6 100644 --- a/src/Interpreters/Session.h +++ b/src/Interpreters/Session.h @@ -77,6 +77,9 @@ public: /// Releases the currently used session ID so it becomes available for reuse by another session. void releaseSessionID(); + /// Closes and removes session + void closeSession(const String & session_id); + private: std::shared_ptr getSessionLog() const; ContextMutablePtr makeQueryContextImpl(const ClientInfo * client_info_to_copy, ClientInfo * client_info_to_move) const; diff --git a/src/Interpreters/SystemLog.cpp b/src/Interpreters/SystemLog.cpp index 9d8547abcf2..78513920236 100644 --- a/src/Interpreters/SystemLog.cpp +++ b/src/Interpreters/SystemLog.cpp @@ -426,6 +426,8 @@ void SystemLog::flushImpl(const std::vector & to_flush, // we need query context to do inserts to target table with MV containing subqueries or joins auto insert_context = Context::createCopy(context); insert_context->makeQueryContext(); + /// We always want to deliver the data to the original table regardless of the MVs + insert_context->setSetting("materialized_views_ignore_errors", true); InterpreterInsertQuery interpreter(query_ptr, insert_context); BlockIO io = interpreter.execute(); diff --git a/src/Interpreters/TemporaryDataOnDisk.cpp b/src/Interpreters/TemporaryDataOnDisk.cpp index e183124cadf..0b56ce29545 100644 --- a/src/Interpreters/TemporaryDataOnDisk.cpp +++ b/src/Interpreters/TemporaryDataOnDisk.cpp @@ -115,7 +115,7 @@ struct TemporaryFileStream::OutputWriter , out_compressed_buf(*out_buf) , out_writer(out_compressed_buf, DBMS_TCP_PROTOCOL_VERSION, header_) { - LOG_TEST(&Poco::Logger::get("TemporaryFileStream"), "Writing to {}", path); + LOG_TEST(&Poco::Logger::get("TemporaryFileStream"), "Writing to temporary file {}", path); } OutputWriter(std::unique_ptr out_buf_, const Block & header_) @@ -124,7 +124,7 @@ struct TemporaryFileStream::OutputWriter , out_writer(out_compressed_buf, DBMS_TCP_PROTOCOL_VERSION, header_) { LOG_TEST(&Poco::Logger::get("TemporaryFileStream"), - "Writing to {}", + "Writing to temporary file {}", static_cast(out_buf.get())->getFileName()); } diff --git a/src/Interpreters/ThreadStatusExt.cpp b/src/Interpreters/ThreadStatusExt.cpp index 84400fc3711..da4f97c1483 100644 --- a/src/Interpreters/ThreadStatusExt.cpp +++ b/src/Interpreters/ThreadStatusExt.cpp @@ -352,8 +352,10 @@ void ThreadStatus::detachQuery(bool exit_if_already_detached, bool thread_exits) thread_group->threads.erase(this); } performance_counters.setParent(&ProfileEvents::global_counters); - memory_tracker.reset(); + flushUntrackedMemory(); + + memory_tracker.reset(); memory_tracker.setParent(thread_group->memory_tracker.getParent()); query_id.clear(); @@ -550,15 +552,16 @@ void CurrentThread::detachQueryIfNotDetached() } -CurrentThread::QueryScope::QueryScope(ContextMutablePtr query_context) +CurrentThread::QueryScope::QueryScope(ContextMutablePtr query_context, std::function fatal_error_callback) { CurrentThread::initializeQuery(); CurrentThread::attachQueryContext(query_context); if (!query_context->hasQueryContext()) query_context->makeQueryContext(); + setFatalErrorCallback(fatal_error_callback); } -CurrentThread::QueryScope::QueryScope(ContextPtr query_context) +CurrentThread::QueryScope::QueryScope(ContextPtr query_context, std::function fatal_error_callback) { if (!query_context->hasQueryContext()) throw Exception( @@ -566,6 +569,7 @@ CurrentThread::QueryScope::QueryScope(ContextPtr query_context) CurrentThread::initializeQuery(); CurrentThread::attachQueryContext(query_context); + setFatalErrorCallback(fatal_error_callback); } void CurrentThread::QueryScope::logPeakMemoryUsage() @@ -585,6 +589,7 @@ CurrentThread::QueryScope::~QueryScope() if (log_peak_memory_usage_in_destructor) logPeakMemoryUsage(); + setFatalErrorCallback({}); CurrentThread::detachQueryIfNotDetached(); } catch (...) diff --git a/src/Interpreters/TransactionLog.cpp b/src/Interpreters/TransactionLog.cpp index 827957b8749..6257e617d4a 100644 --- a/src/Interpreters/TransactionLog.cpp +++ b/src/Interpreters/TransactionLog.cpp @@ -350,7 +350,7 @@ void TransactionLog::tryFinalizeUnknownStateTransactions() /// CSNs must be already loaded, only need to check if the corresponding mapping exists. if (auto csn = getCSN(txn->tid)) { - finalizeCommittedTransaction(txn, csn, state_guard); + finalizeCommittedTransaction(txn.get(), csn, state_guard); } else { @@ -431,7 +431,7 @@ CSN TransactionLog::commitTransaction(const MergeTreeTransactionPtr & txn, bool /// The only thing we can do is to postpone its finalization. { std::lock_guard lock{running_list_mutex}; - unknown_state_list.emplace_back(txn.get(), std::move(state_guard)); + unknown_state_list.emplace_back(txn, std::move(state_guard)); } log_updated_event->set(); if (throw_on_unknown_status) @@ -487,6 +487,7 @@ CSN TransactionLog::finalizeCommittedTransaction(MergeTreeTransaction * txn, CSN } } + txn->afterFinalize(); return allocated_csn; } @@ -523,6 +524,7 @@ void TransactionLog::rollbackTransaction(const MergeTreeTransactionPtr & txn) no } tryWriteEventToSystemLog(log, global_context, TransactionsInfoLogElement::ROLLBACK, txn->tid); + txn->afterFinalize(); } MergeTreeTransactionPtr TransactionLog::tryGetRunningTransaction(const TIDHash & tid) diff --git a/src/Interpreters/TransactionLog.h b/src/Interpreters/TransactionLog.h index 64d02ad3ac5..6e8777d8519 100644 --- a/src/Interpreters/TransactionLog.h +++ b/src/Interpreters/TransactionLog.h @@ -177,7 +177,7 @@ private: /// Transactions that are currently processed TransactionsList running_list TSA_GUARDED_BY(running_list_mutex); /// If we lost connection on attempt to create csn- node then we don't know transaction's state. - using UnknownStateList = std::vector>; + using UnknownStateList = std::vector>; UnknownStateList unknown_state_list TSA_GUARDED_BY(running_list_mutex); UnknownStateList unknown_state_list_loaded TSA_GUARDED_BY(running_list_mutex); /// Ordered list of snapshots that are currently used by some transactions. Needed for background cleanup. diff --git a/src/Interpreters/executeDDLQueryOnCluster.cpp b/src/Interpreters/executeDDLQueryOnCluster.cpp index 60d16eda6ba..c1063e25587 100644 --- a/src/Interpreters/executeDDLQueryOnCluster.cpp +++ b/src/Interpreters/executeDDLQueryOnCluster.cpp @@ -55,7 +55,7 @@ bool isSupportedAlterType(int type) BlockIO executeDDLQueryOnCluster(const ASTPtr & query_ptr_, ContextPtr context, const DDLQueryOnClusterParams & params) { - OpenTelemetry::SpanHolder span(__FUNCTION__); + OpenTelemetry::SpanHolder span(__FUNCTION__, OpenTelemetry::PRODUCER); if (context->getCurrentTransaction() && context->getSettingsRef().throw_on_unsupported_query_inside_transaction) throw Exception(ErrorCodes::NOT_IMPLEMENTED, "ON CLUSTER queries inside transactions are not supported"); diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index 435401796a0..74394879191 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -176,7 +176,7 @@ static void setExceptionStackTrace(QueryLogElement & elem) /// Log exception (with query info) into text log (not into system table). -static void logException(ContextPtr context, QueryLogElement & elem) +static void logException(ContextPtr context, QueryLogElement & elem, bool log_error = true) { String comment; if (!elem.log_comment.empty()) @@ -187,7 +187,7 @@ static void logException(ContextPtr context, QueryLogElement & elem) PreformattedMessage message; message.format_string = elem.exception_format_string; - if (elem.stack_trace.empty()) + if (elem.stack_trace.empty() || !log_error) message.text = fmt::format("{} (from {}){} (in query: {})", elem.exception, context->getClientInfo().current_address.toString(), comment, @@ -201,7 +201,10 @@ static void logException(ContextPtr context, QueryLogElement & elem) toOneLineQuery(elem.query), elem.stack_trace); - LOG_ERROR(&Poco::Logger::get("executeQuery"), message); + if (log_error) + LOG_ERROR(&Poco::Logger::get("executeQuery"), message); + else + LOG_INFO(&Poco::Logger::get("executeQuery"), message); } static void onExceptionBeforeStart( @@ -303,22 +306,6 @@ static void setQuerySpecificSettings(ASTPtr & ast, ContextMutablePtr context) } } -static void applySettingsFromSelectWithUnion(const ASTSelectWithUnionQuery & select_with_union, ContextMutablePtr context) -{ - const ASTs & children = select_with_union.list_of_selects->children; - if (children.empty()) - return; - - // We might have an arbitrarily complex UNION tree, so just give - // up if the last first-order child is not a plain SELECT. - // It is flattened later, when we process UNION ALL/DISTINCT. - const auto * last_select = children.back()->as(); - if (last_select && last_select->settings()) - { - InterpreterSetQuery(last_select->settings(), context).executeForCurrentContext(); - } -} - static std::tuple executeQueryImpl( const char * begin, const char * end, @@ -448,10 +435,24 @@ static std::tuple executeQueryImpl( /// Avoid early destruction of process_list_entry if it was not saved to `res` yet (in case of exception) ProcessList::EntryPtr process_list_entry; BlockIO res; - std::shared_ptr implicit_txn_control{}; + auto implicit_txn_control = std::make_shared(false); String query_database; String query_table; + auto execute_implicit_tcl_query = [implicit_txn_control](const ContextMutablePtr & query_context, ASTTransactionControl::QueryType tcl_type) + { + /// Unset the flag on COMMIT and ROLLBACK + SCOPE_EXIT({ if (tcl_type != ASTTransactionControl::BEGIN) *implicit_txn_control = false; }); + + ASTPtr tcl_ast = std::make_shared(tcl_type); + InterpreterTransactionControlQuery tc(tcl_ast, query_context); + tc.execute(); + + /// Set the flag after successful BIGIN + if (tcl_type == ASTTransactionControl::BEGIN) + *implicit_txn_control = true; + }; + try { if (auto txn = context->getCurrentTransaction()) @@ -466,35 +467,10 @@ static std::tuple executeQueryImpl( /// Interpret SETTINGS clauses as early as possible (before invoking the corresponding interpreter), /// to allow settings to take effect. - if (const auto * select_query = ast->as()) - { - if (auto new_settings = select_query->settings()) - InterpreterSetQuery(new_settings, context).executeForCurrentContext(); - } - else if (const auto * select_with_union_query = ast->as()) - { - applySettingsFromSelectWithUnion(*select_with_union_query, context); - } - else if (const auto * query_with_output = dynamic_cast(ast.get())) - { - if (query_with_output->settings_ast) - InterpreterSetQuery(query_with_output->settings_ast, context).executeForCurrentContext(); + InterpreterSetQuery::applySettingsFromQuery(ast, context); - if (const auto * create_query = ast->as()) - { - if (create_query->select) - { - applySettingsFromSelectWithUnion(create_query->select->as(), context); - } - } - } - else if (auto * insert_query = ast->as()) - { - context->setInsertFormat(insert_query->format); - if (insert_query->settings_ast) - InterpreterSetQuery(insert_query->settings_ast, context).executeForCurrentContext(); + if (auto * insert_query = ast->as()) insert_query->tail = istr; - } setQuerySpecificSettings(ast, context); @@ -661,6 +637,10 @@ static std::tuple executeQueryImpl( } } + bool can_use_query_cache = + settings.allow_experimental_query_cache && settings.use_query_cache + && !ast->as(); + if (!async_insert) { /// We need to start the (implicit) transaction before getting the interpreter as this will get links to the latest snapshots @@ -671,14 +651,7 @@ static std::tuple executeQueryImpl( if (context->isGlobalContext()) throw Exception(ErrorCodes::LOGICAL_ERROR, "Global context cannot create transactions"); - /// If there is no session (which is the default for the HTTP Handler), set up one just for this as it is necessary - /// to control the transaction lifetime - if (!context->hasSessionContext()) - context->makeSessionContext(); - - auto tc = std::make_shared(ast, context); - tc->executeBegin(context->getSessionContext()); - implicit_txn_control = std::move(tc); + execute_implicit_tcl_query(context, ASTTransactionControl::BEGIN); } catch (Exception & e) { @@ -747,7 +720,7 @@ static std::tuple executeQueryImpl( auto query_cache = context->getQueryCache(); bool read_result_from_query_cache = false; /// a query must not read from *and* write to the query cache at the same time if (query_cache != nullptr - && (settings.allow_experimental_query_cache && settings.use_query_cache && settings.enable_reads_from_query_cache) + && (can_use_query_cache && settings.enable_reads_from_query_cache) && res.pipeline.pulling()) { QueryCache::Key key( @@ -768,7 +741,7 @@ static std::tuple executeQueryImpl( /// then add a processor on top of the pipeline which stores the result in the query cache. if (!read_result_from_query_cache && query_cache != nullptr - && settings.allow_experimental_query_cache && settings.use_query_cache && settings.enable_writes_to_query_cache + && can_use_query_cache && settings.enable_writes_to_query_cache && res.pipeline.pulling() && (!astContainsNonDeterministicFunctions(ast, context) || settings.query_cache_store_results_of_queries_with_nondeterministic_functions)) { @@ -936,8 +909,7 @@ static std::tuple executeQueryImpl( auto finish_callback = [elem, context, ast, - allow_experimental_query_cache = settings.allow_experimental_query_cache, - use_query_cache = settings.use_query_cache, + can_use_query_cache = can_use_query_cache, enable_writes_to_query_cache = settings.enable_writes_to_query_cache, query_cache_store_results_of_queries_with_nondeterministic_functions = settings.query_cache_store_results_of_queries_with_nondeterministic_functions, log_queries, @@ -946,6 +918,7 @@ static std::tuple executeQueryImpl( log_processors_profiles = settings.log_processors_profiles, status_info_to_query_log, implicit_txn_control, + execute_implicit_tcl_query, pulling_pipeline = pipeline.pulling(), query_span](QueryPipeline & query_pipeline) mutable { @@ -954,7 +927,7 @@ static std::tuple executeQueryImpl( auto query_cache = context->getQueryCache(); if (query_cache != nullptr && pulling_pipeline - && allow_experimental_query_cache && use_query_cache && enable_writes_to_query_cache + && can_use_query_cache && enable_writes_to_query_cache && (!astContainsNonDeterministicFunctions(ast, context) || query_cache_store_results_of_queries_with_nondeterministic_functions)) { query_pipeline.finalizeWriteInQueryCache(); @@ -995,7 +968,7 @@ static std::tuple executeQueryImpl( { double elapsed_seconds = static_cast(info.elapsed_microseconds) / 1000000.0; double rows_per_second = static_cast(elem.read_rows) / elapsed_seconds; - LOG_INFO( + LOG_DEBUG( &Poco::Logger::get("executeQuery"), "Read {} rows, {} in {} sec., {} rows/sec., {}/sec.", elem.read_rows, @@ -1059,21 +1032,8 @@ static std::tuple executeQueryImpl( } } - if (implicit_txn_control) - { - try - { - implicit_txn_control->executeCommit(context->getSessionContext()); - implicit_txn_control.reset(); - } - catch (const Exception &) - { - /// An exception might happen when trying to commit the transaction. For example we might get an immediate exception - /// because ZK is down and wait_changes_become_visible_after_commit_mode == WAIT_UNKNOWN - implicit_txn_control.reset(); - throw; - } - } + if (*implicit_txn_control) + execute_implicit_tcl_query(context, ASTTransactionControl::COMMIT); } if (query_span) @@ -1101,13 +1061,11 @@ static std::tuple executeQueryImpl( quota(quota), status_info_to_query_log, implicit_txn_control, - query_span]() mutable + execute_implicit_tcl_query, + query_span](bool log_error) mutable { - if (implicit_txn_control) - { - implicit_txn_control->executeRollback(context->getSessionContext()); - implicit_txn_control.reset(); - } + if (*implicit_txn_control) + execute_implicit_tcl_query(context, ASTTransactionControl::ROLLBACK); else if (auto txn = context->getCurrentTransaction()) txn->onException(); @@ -1139,9 +1097,9 @@ static std::tuple executeQueryImpl( elem.query_duration_ms = start_watch.elapsedMilliseconds(); } - if (current_settings.calculate_text_stack_trace) + if (current_settings.calculate_text_stack_trace && log_error) setExceptionStackTrace(elem); - logException(context, elem); + logException(context, elem, log_error); /// In case of exception we log internal queries also if (log_queries && elem.type >= log_queries_min_type && static_cast(elem.query_duration_ms) >= log_queries_min_query_duration_ms) @@ -1176,15 +1134,10 @@ static std::tuple executeQueryImpl( } catch (...) { - if (implicit_txn_control) - { - implicit_txn_control->executeRollback(context->getSessionContext()); - implicit_txn_control.reset(); - } + if (*implicit_txn_control) + execute_implicit_tcl_query(context, ASTTransactionControl::ROLLBACK); else if (auto txn = context->getCurrentTransaction()) - { txn->onException(); - } if (!internal) onExceptionBeforeStart(query_for_logging, context, ast, query_span, start_watch.elapsedMilliseconds()); @@ -1262,7 +1215,7 @@ void executeQuery( /// If not - copy enough data into 'parse_buf'. WriteBufferFromVector> out(parse_buf); - LimitReadBuffer limit(istr, max_query_size + 1, false); + LimitReadBuffer limit(istr, max_query_size + 1, /* trow_exception */ false, /* exact_limit */ {}); copyData(limit, out); out.finalize(); diff --git a/src/Interpreters/getCustomKeyFilterForParallelReplicas.cpp b/src/Interpreters/getCustomKeyFilterForParallelReplicas.cpp new file mode 100644 index 00000000000..2a32d450497 --- /dev/null +++ b/src/Interpreters/getCustomKeyFilterForParallelReplicas.cpp @@ -0,0 +1,134 @@ +#include + +#include +#include +#include +#include + +#include + +#include + +#include + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int ILLEGAL_TYPE_OF_COLUMN_FOR_FILTER; +} + +bool canUseCustomKey(const Settings & settings, const Cluster & cluster, const Context & context) +{ + return settings.max_parallel_replicas > 1 && context.getParallelReplicasMode() == Context::ParallelReplicasMode::CUSTOM_KEY + && cluster.getShardCount() == 1 && cluster.getShardsInfo()[0].getAllNodeCount() > 1; +} + +ASTPtr getCustomKeyFilterForParallelReplica( + size_t replicas_count, + size_t replica_num, + ASTPtr custom_key_ast, + ParallelReplicasCustomKeyFilterType filter_type, + const IStorage & storage, + const ContextPtr & context) +{ + assert(replicas_count > 1); + if (filter_type == ParallelReplicasCustomKeyFilterType::DEFAULT) + { + // first we do modulo with replica count + auto modulo_function = makeASTFunction("positiveModulo", custom_key_ast, std::make_shared(replicas_count)); + + /// then we compare result to the current replica number (offset) + auto equals_function = makeASTFunction("equals", std::move(modulo_function), std::make_shared(replica_num)); + + return equals_function; + } + + assert(filter_type == ParallelReplicasCustomKeyFilterType::RANGE); + + KeyDescription custom_key_description + = KeyDescription::getKeyFromAST(custom_key_ast, storage.getInMemoryMetadataPtr()->columns, context); + + using RelativeSize = boost::rational; + + RelativeSize size_of_universum = 0; + DataTypePtr custom_key_column_type = custom_key_description.data_types[0]; + + size_of_universum = RelativeSize(std::numeric_limits::max()) + RelativeSize(1); + if (custom_key_description.data_types.size() == 1) + { + if (typeid_cast(custom_key_column_type.get())) + size_of_universum = RelativeSize(std::numeric_limits::max()) + RelativeSize(1); + else if (typeid_cast(custom_key_column_type.get())) + size_of_universum = RelativeSize(std::numeric_limits::max()) + RelativeSize(1); + else if (typeid_cast(custom_key_column_type.get())) + size_of_universum = RelativeSize(std::numeric_limits::max()) + RelativeSize(1); + else if (typeid_cast(custom_key_column_type.get())) + size_of_universum = RelativeSize(std::numeric_limits::max()) + RelativeSize(1); + } + + if (size_of_universum == RelativeSize(0)) + throw Exception( + ErrorCodes::ILLEGAL_TYPE_OF_COLUMN_FOR_FILTER, + "Invalid custom key column type: {}. Must be one unsigned integer type", + custom_key_column_type->getName()); + + RelativeSize relative_range_size = RelativeSize(1) / replicas_count; + RelativeSize relative_range_offset = relative_range_size * RelativeSize(replica_num); + + /// Calculate the half-interval of `[lower, upper)` column values. + bool has_lower_limit = false; + bool has_upper_limit = false; + + RelativeSize lower_limit_rational = relative_range_offset * size_of_universum; + RelativeSize upper_limit_rational = (relative_range_offset + relative_range_size) * size_of_universum; + + UInt64 lower = boost::rational_cast(lower_limit_rational); + UInt64 upper = boost::rational_cast(upper_limit_rational); + + if (lower > 0) + has_lower_limit = true; + + if (upper_limit_rational < size_of_universum) + has_upper_limit = true; + + assert(has_lower_limit || has_upper_limit); + + /// Let's add the conditions to cut off something else when the index is scanned again and when the request is processed. + std::shared_ptr lower_function; + std::shared_ptr upper_function; + + if (has_lower_limit) + { + lower_function = makeASTFunction("greaterOrEquals", custom_key_ast, std::make_shared(lower)); + + if (!has_upper_limit) + return lower_function; + } + + if (has_upper_limit) + { + upper_function = makeASTFunction("less", custom_key_ast, std::make_shared(upper)); + + if (!has_lower_limit) + return upper_function; + } + + assert(upper_function && lower_function); + + return makeASTFunction("and", std::move(lower_function), std::move(upper_function)); +} + +ASTPtr parseCustomKeyForTable(const String & custom_key, const Context & context) +{ + /// Try to parse expression + ParserExpression parser; + const auto & settings = context.getSettingsRef(); + return parseQuery( + parser, custom_key.data(), custom_key.data() + custom_key.size(), + "parallel replicas custom key", settings.max_query_size, settings.max_parser_depth); +} + +} diff --git a/src/Interpreters/getCustomKeyFilterForParallelReplicas.h b/src/Interpreters/getCustomKeyFilterForParallelReplicas.h new file mode 100644 index 00000000000..543f1889b32 --- /dev/null +++ b/src/Interpreters/getCustomKeyFilterForParallelReplicas.h @@ -0,0 +1,27 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace DB +{ + +bool canUseCustomKey(const Settings & settings, const Cluster & cluster, const Context & context); + +/// Get AST for filter created from custom_key +/// replica_num is the number of the replica for which we are generating filter starting from 0 +ASTPtr getCustomKeyFilterForParallelReplica( + size_t replicas_count, + size_t replica_num, + ASTPtr custom_key_ast, + ParallelReplicasCustomKeyFilterType filter_type, + const IStorage & storage, + const ContextPtr & context); + +ASTPtr parseCustomKeyForTable(const String & custom_keys, const Context & context); + +} diff --git a/src/Interpreters/inplaceBlockConversions.cpp b/src/Interpreters/inplaceBlockConversions.cpp index 55d2f7ce5a8..5bbd2667f55 100644 --- a/src/Interpreters/inplaceBlockConversions.cpp +++ b/src/Interpreters/inplaceBlockConversions.cpp @@ -61,11 +61,17 @@ void addDefaultRequiredExpressionsRecursively( RequiredSourceColumnsVisitor::Data columns_context; RequiredSourceColumnsVisitor(columns_context).visit(column_default_expr); NameSet required_columns_names = columns_context.requiredColumns(); + auto required_type = std::make_shared(columns.get(required_column_name).type->getName()); - auto expr = makeASTFunction("_CAST", column_default_expr, std::make_shared(columns.get(required_column_name).type->getName())); + auto expr = makeASTFunction("_CAST", column_default_expr, required_type); if (is_column_in_query && convert_null_to_default) + { expr = makeASTFunction("ifNull", std::make_shared(required_column_name), std::move(expr)); + /// ifNull does not respect LowCardinality. + /// It may be fixed later or re-implemented properly for identical types. + expr = makeASTFunction("_CAST", std::move(expr), required_type); + } default_expr_list_accum->children.emplace_back(setAlias(expr, required_column_name)); added_columns.emplace(required_column_name); diff --git a/src/Interpreters/loadMetadata.cpp b/src/Interpreters/loadMetadata.cpp index 837d0f1789d..47ccff57419 100644 --- a/src/Interpreters/loadMetadata.cpp +++ b/src/Interpreters/loadMetadata.cpp @@ -22,6 +22,8 @@ #include #include +#define ORDINARY_TO_ATOMIC_PREFIX ".tmp_convert." + namespace fs = std::filesystem; namespace DB @@ -117,6 +119,37 @@ static void checkUnsupportedVersion(ContextMutablePtr context, const String & da "If so, you should upgrade through intermediate version.", database_name); } +static void checkIncompleteOrdinaryToAtomicConversion(ContextPtr context, const std::map & databases) +{ + if (context->getConfigRef().has("allow_reserved_database_name_tmp_convert")) + return; + + auto convert_flag_path = fs::path(context->getFlagsPath()) / "convert_ordinary_to_atomic"; + if (!fs::exists(convert_flag_path)) + return; + + /// Flag exists. Let's check if we had an unsuccessful conversion attempt previously + for (const auto & db : databases) + { + if (!db.first.starts_with(ORDINARY_TO_ATOMIC_PREFIX)) + continue; + size_t last_dot = db.first.rfind('.'); + if (last_dot <= strlen(ORDINARY_TO_ATOMIC_PREFIX)) + continue; + + String actual_name = db.first.substr(strlen(ORDINARY_TO_ATOMIC_PREFIX), last_dot - strlen(ORDINARY_TO_ATOMIC_PREFIX)); + + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Found a database with special name: {}. " + "Most likely it indicates that conversion of database {} from Ordinary to Atomic " + "was interrupted or failed in the middle. You can add to config.xml " + "or remove convert_ordinary_to_atomic file from flags/ directory, so the server will start forcefully. " + "After starting the server, you can finish conversion manually by moving rest of the tables from {} to {} " + "(using RENAME TABLE) and executing DROP DATABASE {} and RENAME DATABASE {} TO {}", + backQuote(db.first), backQuote(actual_name), backQuote(actual_name), backQuote(db.first), + backQuote(actual_name), backQuote(db.first), backQuote(actual_name)); + } +} + void loadMetadata(ContextMutablePtr context, const String & default_database_name) { Poco::Logger * log = &Poco::Logger::get("loadMetadata"); @@ -168,6 +201,8 @@ void loadMetadata(ContextMutablePtr context, const String & default_database_nam } } + checkIncompleteOrdinaryToAtomicConversion(context, databases); + /// clickhouse-local creates DatabaseMemory as default database by itself /// For clickhouse-server we need create default database bool create_default_db_if_not_exists = !default_database_name.empty(); @@ -324,7 +359,7 @@ static void maybeConvertOrdinaryDatabaseToAtomic(ContextMutablePtr context, cons database_name, fmt::join(permanently_detached_tables, ", ")); } - String tmp_name = fmt::format(".tmp_convert.{}.{}", database_name, thread_local_rng()); + String tmp_name = fmt::format(ORDINARY_TO_ATOMIC_PREFIX"{}.{}", database_name, thread_local_rng()); try { @@ -415,11 +450,13 @@ void convertDatabasesEnginesIfNeed(ContextMutablePtr context) LOG_INFO(&Poco::Logger::get("loadMetadata"), "Found convert_ordinary_to_atomic file in flags directory, " "will try to convert all Ordinary databases to Atomic"); - fs::remove(convert_flag_path); for (const auto & [name, _] : DatabaseCatalog::instance().getDatabases()) if (name != DatabaseCatalog::SYSTEM_DATABASE) maybeConvertOrdinaryDatabaseToAtomic(context, name, /* tables_started */ true); + + LOG_INFO(&Poco::Logger::get("loadMetadata"), "Conversion finished, removing convert_ordinary_to_atomic flag"); + fs::remove(convert_flag_path); } void startupSystemTables() diff --git a/src/Interpreters/tests/gtest_lru_file_cache.cpp b/src/Interpreters/tests/gtest_lru_file_cache.cpp index 93faafb5cea..62aef2441d6 100644 --- a/src/Interpreters/tests/gtest_lru_file_cache.cpp +++ b/src/Interpreters/tests/gtest_lru_file_cache.cpp @@ -9,7 +9,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/src/Loggers/OwnSplitChannel.cpp b/src/Loggers/OwnSplitChannel.cpp index 047f6caffcd..03db198c305 100644 --- a/src/Loggers/OwnSplitChannel.cpp +++ b/src/Loggers/OwnSplitChannel.cpp @@ -56,7 +56,7 @@ void OwnSplitChannel::tryLogSplit(const Poco::Message & msg) /// breaking some functionality because of unexpected "File not /// found" (or similar) error. /// - /// For example StorageDistributedDirectoryMonitor will mark batch + /// For example DistributedAsyncInsertDirectoryQueue will mark batch /// as broken, some MergeTree code can also be affected. /// /// Also note, that we cannot log the exception here, since this diff --git a/src/Parsers/ASTAlterQuery.h b/src/Parsers/ASTAlterQuery.h index 2a48f5bbd9e..1400113fa9c 100644 --- a/src/Parsers/ASTAlterQuery.h +++ b/src/Parsers/ASTAlterQuery.h @@ -256,6 +256,11 @@ protected: void formatQueryImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; bool isOneCommandTypeOnly(const ASTAlterCommand::Type & type) const; + + void forEachPointerToChild(std::function f) override + { + f(reinterpret_cast(&command_list)); + } }; } diff --git a/src/Parsers/ASTBackupQuery.h b/src/Parsers/ASTBackupQuery.h index a3e3a144c72..0201c2b14f9 100644 --- a/src/Parsers/ASTBackupQuery.h +++ b/src/Parsers/ASTBackupQuery.h @@ -94,5 +94,12 @@ public: void formatImpl(const FormatSettings & format, FormatState &, FormatStateStacked) const override; ASTPtr getRewrittenASTWithoutOnCluster(const WithoutOnClusterASTRewriteParams &) const override; QueryKind getQueryKind() const override; + + void forEachPointerToChild(std::function f) override + { + f(reinterpret_cast(&backup_name)); + f(reinterpret_cast(&base_backup_name)); + } }; + } diff --git a/src/Parsers/ASTConstraintDeclaration.h b/src/Parsers/ASTConstraintDeclaration.h index 437aab1a82d..f48d7ef77fe 100644 --- a/src/Parsers/ASTConstraintDeclaration.h +++ b/src/Parsers/ASTConstraintDeclaration.h @@ -25,5 +25,11 @@ public: ASTPtr clone() const override; void formatImpl(const FormatSettings & s, FormatState & state, FormatStateStacked frame) const override; + + void forEachPointerToChild(std::function f) override + { + f(reinterpret_cast(&expr)); + } }; + } diff --git a/src/Parsers/ASTCreateQuery.cpp b/src/Parsers/ASTCreateQuery.cpp index 955ce62b0f7..e28e863c21f 100644 --- a/src/Parsers/ASTCreateQuery.cpp +++ b/src/Parsers/ASTCreateQuery.cpp @@ -91,6 +91,11 @@ public: ASTPtr clone() const override; void formatImpl(const FormatSettings & s, FormatState & state, FormatStateStacked frame) const override; + + void forEachPointerToChild(std::function f) override + { + f(reinterpret_cast(&elem)); + } }; ASTPtr ASTColumnsElement::clone() const diff --git a/src/Parsers/ASTCreateQuery.h b/src/Parsers/ASTCreateQuery.h index 90a15e09369..230996f610e 100644 --- a/src/Parsers/ASTCreateQuery.h +++ b/src/Parsers/ASTCreateQuery.h @@ -32,6 +32,17 @@ public: void formatImpl(const FormatSettings & s, FormatState & state, FormatStateStacked frame) const override; bool isExtendedStorageDefinition() const; + + void forEachPointerToChild(std::function f) override + { + f(reinterpret_cast(&engine)); + f(reinterpret_cast(&partition_by)); + f(reinterpret_cast(&primary_key)); + f(reinterpret_cast(&order_by)); + f(reinterpret_cast(&sample_by)); + f(reinterpret_cast(&ttl_table)); + f(reinterpret_cast(&settings)); + } }; @@ -57,6 +68,16 @@ public: return (!columns || columns->children.empty()) && (!indices || indices->children.empty()) && (!constraints || constraints->children.empty()) && (!projections || projections->children.empty()); } + + void forEachPointerToChild(std::function f) override + { + f(reinterpret_cast(&columns)); + f(reinterpret_cast(&indices)); + f(reinterpret_cast(&primary_key)); + f(reinterpret_cast(&constraints)); + f(reinterpret_cast(&projections)); + f(reinterpret_cast(&primary_key)); + } }; @@ -126,6 +147,19 @@ public: protected: void formatQueryImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; + + void forEachPointerToChild(std::function f) override + { + f(reinterpret_cast(&columns_list)); + f(reinterpret_cast(&inner_storage)); + f(reinterpret_cast(&storage)); + f(reinterpret_cast(&as_table_function)); + f(reinterpret_cast(&select)); + f(reinterpret_cast(&comment)); + f(reinterpret_cast(&table_overrides)); + f(reinterpret_cast(&dictionary_attributes_list)); + f(reinterpret_cast(&dictionary)); + } }; } diff --git a/src/Parsers/ASTDictionary.h b/src/Parsers/ASTDictionary.h index 3611621b8ad..8c332247d52 100644 --- a/src/Parsers/ASTDictionary.h +++ b/src/Parsers/ASTDictionary.h @@ -47,6 +47,11 @@ public: ASTPtr clone() const override; void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; + + void forEachPointerToChild(std::function f) override + { + f(reinterpret_cast(¶meters)); + } }; diff --git a/src/Parsers/ASTExternalDDLQuery.h b/src/Parsers/ASTExternalDDLQuery.h index 7913d44b970..96600b07f29 100644 --- a/src/Parsers/ASTExternalDDLQuery.h +++ b/src/Parsers/ASTExternalDDLQuery.h @@ -41,6 +41,11 @@ public: } QueryKind getQueryKind() const override { return QueryKind::ExternalDDL; } + + void forEachPointerToChild(std::function f) override + { + f(reinterpret_cast(&from)); + } }; } diff --git a/src/Parsers/ASTFunctionWithKeyValueArguments.h b/src/Parsers/ASTFunctionWithKeyValueArguments.h index 67d591dfcdc..75a8ae0415e 100644 --- a/src/Parsers/ASTFunctionWithKeyValueArguments.h +++ b/src/Parsers/ASTFunctionWithKeyValueArguments.h @@ -33,6 +33,11 @@ public: bool hasSecretParts() const override; void updateTreeHashImpl(SipHash & hash_state) const override; + + void forEachPointerToChild(std::function f) override + { + f(reinterpret_cast(&second)); + } }; diff --git a/src/Parsers/ASTIndexDeclaration.h b/src/Parsers/ASTIndexDeclaration.h index e22c1da4489..bd52a611f3f 100644 --- a/src/Parsers/ASTIndexDeclaration.h +++ b/src/Parsers/ASTIndexDeclaration.h @@ -23,6 +23,12 @@ public: ASTPtr clone() const override; void formatImpl(const FormatSettings & s, FormatState & state, FormatStateStacked frame) const override; + + void forEachPointerToChild(std::function f) override + { + f(reinterpret_cast(&expr)); + f(reinterpret_cast(&type)); + } }; } diff --git a/src/Parsers/ASTProjectionDeclaration.h b/src/Parsers/ASTProjectionDeclaration.h index 53c681c3ec1..df7a7c832a6 100644 --- a/src/Parsers/ASTProjectionDeclaration.h +++ b/src/Parsers/ASTProjectionDeclaration.h @@ -18,6 +18,11 @@ public: ASTPtr clone() const override; void formatImpl(const FormatSettings & s, FormatState & state, FormatStateStacked frame) const override; + + void forEachPointerToChild(std::function f) override + { + f(reinterpret_cast(&query)); + } }; } diff --git a/src/Parsers/ASTTableOverrides.h b/src/Parsers/ASTTableOverrides.h index c47260789d8..1df267acaa9 100644 --- a/src/Parsers/ASTTableOverrides.h +++ b/src/Parsers/ASTTableOverrides.h @@ -27,6 +27,12 @@ public: String getID(char) const override { return "TableOverride " + table_name; } ASTPtr clone() const override; void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; + + void forEachPointerToChild(std::function f) override + { + f(reinterpret_cast(&columns)); + f(reinterpret_cast(&storage)); + } }; /// List of table overrides, for example: diff --git a/src/Parsers/Access/ParserCreateUserQuery.cpp b/src/Parsers/Access/ParserCreateUserQuery.cpp index de83c5760c1..c1d0691d305 100644 --- a/src/Parsers/Access/ParserCreateUserQuery.cpp +++ b/src/Parsers/Access/ParserCreateUserQuery.cpp @@ -17,7 +17,7 @@ #include #include #include "config.h" -#include +#include #if USE_SSL # include # include diff --git a/src/Parsers/ExpressionElementParsers.cpp b/src/Parsers/ExpressionElementParsers.cpp index 72353a42a87..a6354cd0e81 100644 --- a/src/Parsers/ExpressionElementParsers.cpp +++ b/src/Parsers/ExpressionElementParsers.cpp @@ -2151,8 +2151,9 @@ bool ParserTTLElement::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) ParserKeyword s_set("SET"); ParserKeyword s_recompress("RECOMPRESS"); ParserKeyword s_codec("CODEC"); - ParserToken s_comma(TokenType::Comma); - ParserToken s_eq(TokenType::Equals); + ParserKeyword s_materialize("MATERIALIZE"); + ParserKeyword s_remove("REMOVE"); + ParserKeyword s_modify("MODIFY"); ParserIdentifier parser_identifier; ParserStringLiteral parser_string_literal; @@ -2160,8 +2161,11 @@ bool ParserTTLElement::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) ParserExpressionList parser_keys_list(false); ParserCodec parser_codec; - ParserList parser_assignment_list( - std::make_unique(), std::make_unique(TokenType::Comma)); + if (s_materialize.checkWithoutMoving(pos, expected) || + s_remove.checkWithoutMoving(pos, expected) || + s_modify.checkWithoutMoving(pos, expected)) + + return false; ASTPtr ttl_expr; if (!parser_exp.parse(pos, ttl_expr, expected)) @@ -2219,6 +2223,9 @@ bool ParserTTLElement::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) if (s_set.ignore(pos)) { + ParserList parser_assignment_list( + std::make_unique(), std::make_unique(TokenType::Comma)); + if (!parser_assignment_list.parse(pos, group_by_assignments, expected)) return false; } diff --git a/src/Parsers/FieldFromAST.cpp b/src/Parsers/FieldFromAST.cpp index 3cd10c1cf80..a81bf45a8be 100644 --- a/src/Parsers/FieldFromAST.cpp +++ b/src/Parsers/FieldFromAST.cpp @@ -6,6 +6,7 @@ #include #include #include +#include namespace DB @@ -31,42 +32,64 @@ bool FieldFromASTImpl::isSecret() const return isDiskFunction(ast); } +class DiskConfigurationMasker +{ +public: + struct Data {}; + + static bool needChildVisit(const ASTPtr &, const ASTPtr &) { return true; } + + static void visit(ASTPtr & ast, Data &) + { + if (isDiskFunction(ast)) + { + const auto & disk_function = assert_cast(*ast); + const auto * disk_function_args_expr = assert_cast(disk_function.arguments.get()); + const auto & disk_function_args = disk_function_args_expr->children; + + auto is_secret_arg = [](const std::string & arg_name) + { + /// We allow to not hide type of the disk, e.g. disk(type = s3, ...) + /// and also nested disk, e.g. disk = 'disk_name' + return arg_name != "type" && arg_name != "disk"; + }; + + for (const auto & arg : disk_function_args) + { + auto * setting_function = arg->as(); + if (!setting_function || setting_function->name != "equals") + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Bad format: expected equals function"); + + auto * function_args_expr = assert_cast(setting_function->arguments.get()); + if (!function_args_expr) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Bad format: expected arguments"); + + auto & function_args = function_args_expr->children; + if (function_args.empty()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Bad format: expected non zero number of arguments"); + + auto * key_identifier = function_args[0]->as(); + if (!key_identifier) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Bad format: expected Identifier"); + + const std::string & key = key_identifier->name(); + if (is_secret_arg(key)) + function_args[1] = std::make_shared("[HIDDEN]"); + } + } + } +}; + +/// Visits children first. +using HideDiskConfigurationVisitor = InDepthNodeVisitor; + String FieldFromASTImpl::toString(bool show_secrets) const { if (!show_secrets && isDiskFunction(ast)) { auto hidden = ast->clone(); - const auto & disk_function = assert_cast(*hidden); - const auto * disk_function_args_expr = assert_cast(disk_function.arguments.get()); - const auto & disk_function_args = disk_function_args_expr->children; - - auto is_secret_arg = [](const std::string & arg_name) - { - return arg_name != "type"; - }; - - for (const auto & arg : disk_function_args) - { - auto * setting_function = arg->as(); - if (!setting_function || setting_function->name != "equals") - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Bad format: expected equals function"); - - auto * function_args_expr = assert_cast(setting_function->arguments.get()); - if (!function_args_expr) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Bad format: expected arguments"); - - auto & function_args = function_args_expr->children; - if (function_args.empty()) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Bad format: expected non zero number of arguments"); - - auto * key_identifier = function_args[0]->as(); - if (!key_identifier) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Bad format: expected Identifier"); - - const std::string & key = key_identifier->name(); - if (is_secret_arg(key)) - function_args[1] = std::make_shared("[HIDDEN]"); - } + HideDiskConfigurationVisitor::Data data{}; + HideDiskConfigurationVisitor{data}.visit(hidden); return serializeAST(*hidden); } diff --git a/src/Parsers/IAST.h b/src/Parsers/IAST.h index 627b1174b33..5928506aa5b 100644 --- a/src/Parsers/IAST.h +++ b/src/Parsers/IAST.h @@ -175,6 +175,16 @@ public: field = nullptr; } + /// After changing one of `children` elements, update the corresponding member pointer if needed. + void updatePointerToChild(void * old_ptr, void * new_ptr) + { + forEachPointerToChild([old_ptr, new_ptr](void ** ptr) mutable + { + if (*ptr == old_ptr) + *ptr = new_ptr; + }); + } + /// Convert to a string. /// Format settings. @@ -295,6 +305,10 @@ public: protected: bool childrenHaveSecretParts() const; + /// Some AST classes have naked pointers to children elements as members. + /// This method allows to iterate over them. + virtual void forEachPointerToChild(std::function) {} + private: size_t checkDepthImpl(size_t max_depth) const; diff --git a/src/Parsers/MySQL/ASTAlterCommand.h b/src/Parsers/MySQL/ASTAlterCommand.h index f097ed71219..87b665ec6a5 100644 --- a/src/Parsers/MySQL/ASTAlterCommand.h +++ b/src/Parsers/MySQL/ASTAlterCommand.h @@ -80,6 +80,15 @@ protected: { throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method formatImpl is not supported by MySQLParser::ASTAlterCommand."); } + + void forEachPointerToChild(std::function f) override + { + f(reinterpret_cast(&index_decl)); + f(reinterpret_cast(&default_expression)); + f(reinterpret_cast(&additional_columns)); + f(reinterpret_cast(&order_by_columns)); + f(reinterpret_cast(&properties)); + } }; class ParserAlterCommand : public IParserBase diff --git a/src/Parsers/MySQL/ASTCreateDefines.h b/src/Parsers/MySQL/ASTCreateDefines.h index 3d2a79568ab..7c23d1cb87f 100644 --- a/src/Parsers/MySQL/ASTCreateDefines.h +++ b/src/Parsers/MySQL/ASTCreateDefines.h @@ -31,6 +31,13 @@ protected: { throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method formatImpl is not supported by MySQLParser::ASTCreateDefines."); } + + void forEachPointerToChild(std::function f) override + { + f(reinterpret_cast(&columns)); + f(reinterpret_cast(&indices)); + f(reinterpret_cast(&constraints)); + } }; class ParserCreateDefines : public IParserBase @@ -44,4 +51,3 @@ protected: } } - diff --git a/src/Planner/Planner.cpp b/src/Planner/Planner.cpp index 2ce470d9ecf..2242bf92e6b 100644 --- a/src/Planner/Planner.cpp +++ b/src/Planner/Planner.cpp @@ -214,9 +214,14 @@ public: { /// Constness of limit is validated during query analysis stage limit_length = query_node.getLimit()->as().getValue().safeGet(); - } - if (query_node.hasOffset()) + if (query_node.hasOffset() && limit_length) + { + /// Constness of offset is validated during query analysis stage + limit_offset = query_node.getOffset()->as().getValue().safeGet(); + } + } + else if (query_node.hasOffset()) { /// Constness of offset is validated during query analysis stage limit_offset = query_node.getOffset()->as().getValue().safeGet(); @@ -390,7 +395,11 @@ void addMergingAggregatedStep(QueryPlan & query_plan, * but it can work more slowly. */ - Aggregator::Params params(aggregation_analysis_result.aggregation_keys, + auto keys = aggregation_analysis_result.aggregation_keys; + if (!aggregation_analysis_result.grouping_sets_parameters_list.empty()) + keys.insert(keys.begin(), "__grouping_set"); + + Aggregator::Params params(keys, aggregation_analysis_result.aggregate_descriptions, query_analysis_result.aggregate_overflow_row, settings.max_threads, diff --git a/src/Planner/PlannerActionsVisitor.cpp b/src/Planner/PlannerActionsVisitor.cpp index 614e5c03bc1..ac524a011a9 100644 --- a/src/Planner/PlannerActionsVisitor.cpp +++ b/src/Planner/PlannerActionsVisitor.cpp @@ -132,7 +132,7 @@ public: } template - const ActionsDAG::Node * addFunctionIfNecessary(const std::string & node_name, ActionsDAG::NodeRawConstPtrs children, FunctionOrOverloadResolver function) + const ActionsDAG::Node * addFunctionIfNecessary(const std::string & node_name, ActionsDAG::NodeRawConstPtrs children, const FunctionOrOverloadResolver & function) { auto it = node_name_to_node.find(node_name); if (it != node_name_to_node.end()) @@ -339,7 +339,7 @@ PlannerActionsVisitorImpl::NodeNameAndNodeMinLevel PlannerActionsVisitorImpl::vi actions_stack.pop_back(); // TODO: Pass IFunctionBase here not FunctionCaptureOverloadResolver. - actions_stack[level].addFunctionIfNecessary(lambda_node_name, std::move(lambda_children), std::move(function_capture)); + actions_stack[level].addFunctionIfNecessary(lambda_node_name, std::move(lambda_children), function_capture); size_t actions_stack_size = actions_stack.size(); for (size_t i = level + 1; i < actions_stack_size; ++i) @@ -501,7 +501,7 @@ PlannerActionsVisitorImpl::NodeNameAndNodeMinLevel PlannerActionsVisitorImpl::vi } else { - actions_stack[level].addFunctionIfNecessary(function_node_name, children, function_node.getFunction()); + actions_stack[level].addFunctionIfNecessary(function_node_name, children, function_node); } size_t actions_stack_size = actions_stack.size(); diff --git a/src/Planner/PlannerContext.cpp b/src/Planner/PlannerContext.cpp index 9f4a489bf5f..59ae0f20fac 100644 --- a/src/Planner/PlannerContext.cpp +++ b/src/Planner/PlannerContext.cpp @@ -45,7 +45,7 @@ bool GlobalPlannerContext::hasColumnIdentifier(const ColumnIdentifier & column_i return column_identifiers.contains(column_identifier); } -PlannerContext::PlannerContext(ContextPtr query_context_, GlobalPlannerContextPtr global_planner_context_) +PlannerContext::PlannerContext(ContextMutablePtr query_context_, GlobalPlannerContextPtr global_planner_context_) : query_context(std::move(query_context_)) , global_planner_context(std::move(global_planner_context_)) {} diff --git a/src/Planner/PlannerContext.h b/src/Planner/PlannerContext.h index 63874bf7ab9..e47198bfe5f 100644 --- a/src/Planner/PlannerContext.h +++ b/src/Planner/PlannerContext.h @@ -88,16 +88,22 @@ class PlannerContext { public: /// Create planner context with query context and global planner context - PlannerContext(ContextPtr query_context_, GlobalPlannerContextPtr global_planner_context_); + PlannerContext(ContextMutablePtr query_context_, GlobalPlannerContextPtr global_planner_context_); /// Get planner context query context - const ContextPtr & getQueryContext() const + ContextPtr getQueryContext() const { return query_context; } - /// Get planner context query context - ContextPtr & getQueryContext() + /// Get planner context mutable query context + const ContextMutablePtr & getMutableQueryContext() const + { + return query_context; + } + + /// Get planner context mutable query context + ContextMutablePtr & getMutableQueryContext() { return query_context; } @@ -137,12 +143,18 @@ public: */ TableExpressionData * getTableExpressionDataOrNull(const QueryTreeNodePtr & table_expression_node); - /// Get table expression node to data read only map + /// Get table expression node to data map const std::unordered_map & getTableExpressionNodeToData() const { return table_expression_node_to_data; } + /// Get table expression node to data map + std::unordered_map & getTableExpressionNodeToData() + { + return table_expression_node_to_data; + } + /** Get column node identifier. * For column node source check if table expression data is registered. * If table expression data is not registered exception is thrown. @@ -184,7 +196,7 @@ public: private: /// Query context - ContextPtr query_context; + ContextMutablePtr query_context; /// Global planner context GlobalPlannerContextPtr global_planner_context; diff --git a/src/Planner/PlannerExpressionAnalysis.cpp b/src/Planner/PlannerExpressionAnalysis.cpp index 0e97e69cd67..11444503c5f 100644 --- a/src/Planner/PlannerExpressionAnalysis.cpp +++ b/src/Planner/PlannerExpressionAnalysis.cpp @@ -1,6 +1,7 @@ #include #include +#include #include #include @@ -33,16 +34,13 @@ namespace * It is client responsibility to update filter analysis result if filter column must be removed after chain is finalized. */ FilterAnalysisResult analyzeFilter(const QueryTreeNodePtr & filter_expression_node, - const ColumnsWithTypeAndName & join_tree_input_columns, + const ColumnsWithTypeAndName & input_columns, const PlannerContextPtr & planner_context, ActionsChain & actions_chain) { - const auto * chain_available_output_columns = actions_chain.getLastStepAvailableOutputColumnsOrNull(); - const auto & filter_input = chain_available_output_columns ? *chain_available_output_columns : join_tree_input_columns; - FilterAnalysisResult result; - result.filter_actions = buildActionsDAGFromExpressionNode(filter_expression_node, filter_input, planner_context); + result.filter_actions = buildActionsDAGFromExpressionNode(filter_expression_node, input_columns, planner_context); result.filter_column_name = result.filter_actions->getOutputs().at(0)->result_name; actions_chain.addStep(std::make_unique(result.filter_actions)); @@ -53,7 +51,7 @@ FilterAnalysisResult analyzeFilter(const QueryTreeNodePtr & filter_expression_no * Actions before aggregation are added into actions chain, if result is not null optional. */ std::optional analyzeAggregation(const QueryTreeNodePtr & query_tree, - const ColumnsWithTypeAndName & join_tree_input_columns, + const ColumnsWithTypeAndName & input_columns, const PlannerContextPtr & planner_context, ActionsChain & actions_chain) { @@ -69,10 +67,7 @@ std::optional analyzeAggregation(const QueryTreeNodeP Names aggregation_keys; - const auto * chain_available_output_columns = actions_chain.getLastStepAvailableOutputColumnsOrNull(); - const auto & group_by_input = chain_available_output_columns ? *chain_available_output_columns : join_tree_input_columns; - - ActionsDAGPtr before_aggregation_actions = std::make_shared(group_by_input); + ActionsDAGPtr before_aggregation_actions = std::make_shared(input_columns); before_aggregation_actions->getOutputs().clear(); std::unordered_set before_aggregation_actions_output_node_names; @@ -83,6 +78,8 @@ std::optional analyzeAggregation(const QueryTreeNodeP PlannerActionsVisitor actions_visitor(planner_context); /// Add expressions from GROUP BY + bool group_by_use_nulls = planner_context->getQueryContext()->getSettingsRef().group_by_use_nulls && + (query_node.isGroupByWithGroupingSets() || query_node.isGroupByWithRollup() || query_node.isGroupByWithCube()); if (query_node.hasGroupBy()) { @@ -107,6 +104,8 @@ std::optional analyzeAggregation(const QueryTreeNodeP if (before_aggregation_actions_output_node_names.contains(expression_dag_node->result_name)) continue; + auto expression_type_after_aggregation = group_by_use_nulls ? makeNullableSafe(expression_dag_node->result_type) : expression_dag_node->result_type; + available_columns_after_aggregation.emplace_back(nullptr, expression_type_after_aggregation, expression_dag_node->result_name); aggregation_keys.push_back(expression_dag_node->result_name); before_aggregation_actions->getOutputs().push_back(expression_dag_node); before_aggregation_actions_output_node_names.insert(expression_dag_node->result_name); @@ -150,6 +149,8 @@ std::optional analyzeAggregation(const QueryTreeNodeP if (before_aggregation_actions_output_node_names.contains(expression_dag_node->result_name)) continue; + auto expression_type_after_aggregation = group_by_use_nulls ? makeNullableSafe(expression_dag_node->result_type) : expression_dag_node->result_type; + available_columns_after_aggregation.emplace_back(nullptr, expression_type_after_aggregation, expression_dag_node->result_name); aggregation_keys.push_back(expression_dag_node->result_name); before_aggregation_actions->getOutputs().push_back(expression_dag_node); before_aggregation_actions_output_node_names.insert(expression_dag_node->result_name); @@ -157,9 +158,6 @@ std::optional analyzeAggregation(const QueryTreeNodeP } } - for (auto & node : before_aggregation_actions->getOutputs()) - available_columns_after_aggregation.emplace_back(nullptr, node->result_type, node->result_name); - /// Add expressions from aggregate functions arguments for (auto & aggregate_function_node : aggregate_function_nodes) @@ -208,7 +206,7 @@ std::optional analyzeAggregation(const QueryTreeNodeP * Actions before window functions are added into actions chain, if result is not null optional. */ std::optional analyzeWindow(const QueryTreeNodePtr & query_tree, - const ColumnsWithTypeAndName & join_tree_input_columns, + const ColumnsWithTypeAndName & input_columns, const PlannerContextPtr & planner_context, ActionsChain & actions_chain) { @@ -218,12 +216,9 @@ std::optional analyzeWindow(const QueryTreeNodePtr & query auto window_descriptions = extractWindowDescriptions(window_function_nodes, *planner_context); - const auto * chain_available_output_columns = actions_chain.getLastStepAvailableOutputColumnsOrNull(); - const auto & window_input = chain_available_output_columns ? *chain_available_output_columns : join_tree_input_columns; - PlannerActionsVisitor actions_visitor(planner_context); - ActionsDAGPtr before_window_actions = std::make_shared(window_input); + ActionsDAGPtr before_window_actions = std::make_shared(input_columns); before_window_actions->getOutputs().clear(); std::unordered_set before_window_actions_output_node_names; @@ -298,13 +293,11 @@ std::optional analyzeWindow(const QueryTreeNodePtr & query * It is client responsibility to update projection analysis result with project names actions after chain is finalized. */ ProjectionAnalysisResult analyzeProjection(const QueryNode & query_node, - const ColumnsWithTypeAndName & join_tree_input_columns, + const ColumnsWithTypeAndName & input_columns, const PlannerContextPtr & planner_context, ActionsChain & actions_chain) { - const auto * chain_available_output_columns = actions_chain.getLastStepAvailableOutputColumnsOrNull(); - const auto & projection_input = chain_available_output_columns ? *chain_available_output_columns : join_tree_input_columns; - auto projection_actions = buildActionsDAGFromExpressionNode(query_node.getProjectionNode(), projection_input, planner_context); + auto projection_actions = buildActionsDAGFromExpressionNode(query_node.getProjectionNode(), input_columns, planner_context); auto projection_columns = query_node.getProjectionColumns(); size_t projection_columns_size = projection_columns.size(); @@ -347,14 +340,11 @@ ProjectionAnalysisResult analyzeProjection(const QueryNode & query_node, * Actions before sort are added into actions chain. */ SortAnalysisResult analyzeSort(const QueryNode & query_node, - const ColumnsWithTypeAndName & join_tree_input_columns, + const ColumnsWithTypeAndName & input_columns, const PlannerContextPtr & planner_context, ActionsChain & actions_chain) { - const auto * chain_available_output_columns = actions_chain.getLastStepAvailableOutputColumnsOrNull(); - const auto & order_by_input = chain_available_output_columns ? *chain_available_output_columns : join_tree_input_columns; - - ActionsDAGPtr before_sort_actions = std::make_shared(order_by_input); + ActionsDAGPtr before_sort_actions = std::make_shared(input_columns); auto & before_sort_actions_outputs = before_sort_actions->getOutputs(); before_sort_actions_outputs.clear(); @@ -437,14 +427,12 @@ SortAnalysisResult analyzeSort(const QueryNode & query_node, * Actions before limit by are added into actions chain. */ LimitByAnalysisResult analyzeLimitBy(const QueryNode & query_node, - const ColumnsWithTypeAndName & join_tree_input_columns, + const ColumnsWithTypeAndName & input_columns, const PlannerContextPtr & planner_context, const NameSet & required_output_nodes_names, ActionsChain & actions_chain) { - const auto * chain_available_output_columns = actions_chain.getLastStepAvailableOutputColumnsOrNull(); - const auto & limit_by_input = chain_available_output_columns ? *chain_available_output_columns : join_tree_input_columns; - auto before_limit_by_actions = buildActionsDAGFromExpressionNode(query_node.getLimitByNode(), limit_by_input, planner_context); + auto before_limit_by_actions = buildActionsDAGFromExpressionNode(query_node.getLimitByNode(), input_columns, planner_context); NameSet limit_by_column_names_set; Names limit_by_column_names; @@ -482,29 +470,42 @@ PlannerExpressionsAnalysisResult buildExpressionAnalysisResult(const QueryTreeNo std::optional where_analysis_result_optional; std::optional where_action_step_index_optional; + ColumnsWithTypeAndName current_output_columns = join_tree_input_columns; + if (query_node.hasWhere()) { - where_analysis_result_optional = analyzeFilter(query_node.getWhere(), join_tree_input_columns, planner_context, actions_chain); + where_analysis_result_optional = analyzeFilter(query_node.getWhere(), current_output_columns, planner_context, actions_chain); where_action_step_index_optional = actions_chain.getLastStepIndex(); + current_output_columns = actions_chain.getLastStepAvailableOutputColumns(); } - auto aggregation_analysis_result_optional = analyzeAggregation(query_tree, join_tree_input_columns, planner_context, actions_chain); + auto aggregation_analysis_result_optional = analyzeAggregation(query_tree, current_output_columns, planner_context, actions_chain); + if (aggregation_analysis_result_optional) + current_output_columns = actions_chain.getLastStepAvailableOutputColumns(); std::optional having_analysis_result_optional; std::optional having_action_step_index_optional; if (query_node.hasHaving()) { - having_analysis_result_optional = analyzeFilter(query_node.getHaving(), join_tree_input_columns, planner_context, actions_chain); + having_analysis_result_optional = analyzeFilter(query_node.getHaving(), current_output_columns, planner_context, actions_chain); having_action_step_index_optional = actions_chain.getLastStepIndex(); + current_output_columns = actions_chain.getLastStepAvailableOutputColumns(); } - auto window_analysis_result_optional = analyzeWindow(query_tree, join_tree_input_columns, planner_context, actions_chain); - auto projection_analysis_result = analyzeProjection(query_node, join_tree_input_columns, planner_context, actions_chain); + auto window_analysis_result_optional = analyzeWindow(query_tree, current_output_columns, planner_context, actions_chain); + if (window_analysis_result_optional) + current_output_columns = actions_chain.getLastStepAvailableOutputColumns(); + + auto projection_analysis_result = analyzeProjection(query_node, current_output_columns, planner_context, actions_chain); + current_output_columns = actions_chain.getLastStepAvailableOutputColumns(); std::optional sort_analysis_result_optional; if (query_node.hasOrderBy()) - sort_analysis_result_optional = analyzeSort(query_node, join_tree_input_columns, planner_context, actions_chain); + { + sort_analysis_result_optional = analyzeSort(query_node, current_output_columns, planner_context, actions_chain); + current_output_columns = actions_chain.getLastStepAvailableOutputColumns(); + } std::optional limit_by_analysis_result_optional; @@ -526,14 +527,15 @@ PlannerExpressionsAnalysisResult buildExpressionAnalysisResult(const QueryTreeNo } limit_by_analysis_result_optional = analyzeLimitBy(query_node, - join_tree_input_columns, + current_output_columns, planner_context, required_output_nodes_names, actions_chain); + current_output_columns = actions_chain.getLastStepAvailableOutputColumns(); } const auto * chain_available_output_columns = actions_chain.getLastStepAvailableOutputColumnsOrNull(); - auto project_names_input = chain_available_output_columns ? *chain_available_output_columns : join_tree_input_columns; + auto project_names_input = chain_available_output_columns ? *chain_available_output_columns : current_output_columns; bool has_with_fill = sort_analysis_result_optional.has_value() && sort_analysis_result_optional->has_with_fill; /** If there is WITH FILL we must use non constant projection columns. diff --git a/src/Planner/PlannerJoinTree.cpp b/src/Planner/PlannerJoinTree.cpp index d6235208063..a48cceebfb6 100644 --- a/src/Planner/PlannerJoinTree.cpp +++ b/src/Planner/PlannerJoinTree.cpp @@ -1,5 +1,7 @@ #include +#include +#include #include #include @@ -19,6 +21,8 @@ #include #include #include +#include +#include #include #include @@ -27,6 +31,7 @@ #include #include #include +#include #include #include @@ -40,6 +45,10 @@ #include #include +#include +#include +#include + namespace DB { @@ -143,17 +152,181 @@ NameAndTypePair chooseSmallestColumnToReadFromStorage(const StoragePtr & storage return result; } -JoinTreeQueryPlan buildQueryPlanForTableExpression(const QueryTreeNodePtr & table_expression, - const SelectQueryInfo & select_query_info, - const SelectQueryOptions & select_query_options, - PlannerContextPtr & planner_context, - bool is_single_table_expression) +bool applyTrivialCountIfPossible( + QueryPlan & query_plan, + const TableNode & table_node, + const QueryTreeNodePtr & query_tree, + const ContextPtr & query_context, + const Names & columns_names) +{ + const auto & settings = query_context->getSettingsRef(); + if (!settings.optimize_trivial_count_query) + return false; + + /// can't apply if FINAL + if (table_node.getTableExpressionModifiers().has_value() && table_node.getTableExpressionModifiers()->hasFinal()) + return false; + + auto & main_query_node = query_tree->as(); + if (main_query_node.hasGroupBy()) + return false; + + const auto & storage = table_node.getStorage(); + if (!storage || storage->hasLightweightDeletedMask()) + return false; + + if (settings.max_parallel_replicas > 1 || settings.allow_experimental_query_deduplication + || settings.empty_result_for_aggregation_by_empty_set) + return false; + + QueryTreeNodes aggregates = collectAggregateFunctionNodes(query_tree); + if (aggregates.size() != 1) + return false; + + const auto & function_node = aggregates.front().get()->as(); + chassert(function_node.getAggregateFunction() != nullptr); + const auto * count_func = typeid_cast(function_node.getAggregateFunction().get()); + if (!count_func) + return false; + + /// get number of rows + std::optional num_rows{}; + /// Transaction check here is necessary because + /// MergeTree maintains total count for all parts in Active state and it simply returns that number for trivial select count() from table query. + /// But if we have current transaction, then we should return number of rows in current snapshot (that may include parts in Outdated state), + /// so we have to use totalRowsByPartitionPredicate() instead of totalRows even for trivial query + /// See https://github.com/ClickHouse/ClickHouse/pull/24258/files#r828182031 + if (!main_query_node.hasPrewhere() && !main_query_node.hasWhere() && !query_context->getCurrentTransaction()) + { + num_rows = storage->totalRows(settings); + } + // TODO: + // else // It's possible to optimize count() given only partition predicates + // { + // SelectQueryInfo temp_query_info; + // temp_query_info.query = query_ptr; + // temp_query_info.syntax_analyzer_result = syntax_analyzer_result; + // temp_query_info.prepared_sets = query_analyzer->getPreparedSets(); + // num_rows = storage->totalRowsByPartitionPredicate(temp_query_info, context); + // } + + if (!num_rows) + return false; + + /// set aggregation state + const AggregateFunctionCount & agg_count = *count_func; + std::vector state(agg_count.sizeOfData()); + AggregateDataPtr place = state.data(); + agg_count.create(place); + SCOPE_EXIT_MEMORY_SAFE(agg_count.destroy(place)); + agg_count.set(place, num_rows.value()); + + auto column = ColumnAggregateFunction::create(function_node.getAggregateFunction()); + column->insertFrom(place); + + /// get count() argument type + DataTypes argument_types; + argument_types.reserve(columns_names.size()); + { + const Block source_header = table_node.getStorageSnapshot()->getSampleBlockForColumns(columns_names); + for (const auto & column_name : columns_names) + argument_types.push_back(source_header.getByName(column_name).type); + } + + Block block_with_count{ + {std::move(column), + std::make_shared(function_node.getAggregateFunction(), argument_types, Array{}), + columns_names.front()}}; + + auto source = std::make_shared(block_with_count); + auto prepared_count = std::make_unique(Pipe(std::move(source))); + prepared_count->setStepDescription("Optimized trivial count"); + query_plan.addStep(std::move(prepared_count)); + + return true; +} + +void prepareBuildQueryPlanForTableExpression(const QueryTreeNodePtr & table_expression, PlannerContextPtr & planner_context) { const auto & query_context = planner_context->getQueryContext(); const auto & settings = query_context->getSettingsRef(); + auto & table_expression_data = planner_context->getTableExpressionDataOrThrow(table_expression); + auto columns_names = table_expression_data.getColumnNames(); + + auto * table_node = table_expression->as(); + auto * table_function_node = table_expression->as(); + auto * query_node = table_expression->as(); + auto * union_node = table_expression->as(); + + /** The current user must have the SELECT privilege. + * We do not check access rights for table functions because they have been already checked in ITableFunction::execute(). + */ + if (table_node) + { + auto column_names_with_aliases = columns_names; + const auto & alias_columns_names = table_expression_data.getAliasColumnsNames(); + column_names_with_aliases.insert(column_names_with_aliases.end(), alias_columns_names.begin(), alias_columns_names.end()); + checkAccessRights(*table_node, column_names_with_aliases, query_context); + } + + if (columns_names.empty()) + { + NameAndTypePair additional_column_to_read; + + if (table_node || table_function_node) + { + const auto & storage = table_node ? table_node->getStorage() : table_function_node->getStorage(); + const auto & storage_snapshot = table_node ? table_node->getStorageSnapshot() : table_function_node->getStorageSnapshot(); + additional_column_to_read = chooseSmallestColumnToReadFromStorage(storage, storage_snapshot); + + } + else if (query_node || union_node) + { + const auto & projection_columns = query_node ? query_node->getProjectionColumns() : union_node->computeProjectionColumns(); + NamesAndTypesList projection_columns_list(projection_columns.begin(), projection_columns.end()); + additional_column_to_read = ExpressionActions::getSmallestColumn(projection_columns_list); + } + else + { + throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected table, table function, query or union. Actual {}", + table_expression->formatASTForErrorMessage()); + } + + auto & global_planner_context = planner_context->getGlobalPlannerContext(); + const auto & column_identifier = global_planner_context->createColumnIdentifier(additional_column_to_read, table_expression); + columns_names.push_back(additional_column_to_read.name); + table_expression_data.addColumn(additional_column_to_read, column_identifier); + } + + /// Limitation on the number of columns to read + if (settings.max_columns_to_read && columns_names.size() > settings.max_columns_to_read) + throw Exception(ErrorCodes::TOO_MANY_COLUMNS, + "Limit for number of columns to read exceeded. Requested: {}, maximum: {}", + columns_names.size(), + settings.max_columns_to_read); +} + +JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expression, + const SelectQueryInfo & select_query_info, + const SelectQueryOptions & select_query_options, + PlannerContextPtr & planner_context, + bool is_single_table_expression, + bool wrap_read_columns_in_subquery) +{ + const auto & query_context = planner_context->getQueryContext(); + const auto & settings = query_context->getSettingsRef(); + + auto & table_expression_data = planner_context->getTableExpressionDataOrThrow(table_expression); + QueryProcessingStage::Enum from_stage = QueryProcessingStage::Enum::FetchColumns; + if (wrap_read_columns_in_subquery) + { + auto columns = table_expression_data.getColumns(); + table_expression = buildSubqueryToReadColumnsFromTableExpression(columns, table_expression, query_context); + } + auto * table_node = table_expression->as(); auto * table_function_node = table_expression->as(); auto * query_node = table_expression->as(); @@ -161,8 +334,6 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(const QueryTreeNodePtr & tabl QueryPlan query_plan; - auto & table_expression_data = planner_context->getTableExpressionDataOrThrow(table_expression); - if (table_node || table_function_node) { const auto & storage = table_node ? table_node->getStorage() : table_function_node->getStorage(); @@ -259,32 +430,6 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(const QueryTreeNodePtr & tabl auto columns_names = table_expression_data.getColumnNames(); - /** The current user must have the SELECT privilege. - * We do not check access rights for table functions because they have been already checked in ITableFunction::execute(). - */ - if (table_node) - { - auto column_names_with_aliases = columns_names; - const auto & alias_columns_names = table_expression_data.getAliasColumnsNames(); - column_names_with_aliases.insert(column_names_with_aliases.end(), alias_columns_names.begin(), alias_columns_names.end()); - checkAccessRights(*table_node, column_names_with_aliases, planner_context->getQueryContext()); - } - - /// Limitation on the number of columns to read - if (settings.max_columns_to_read && columns_names.size() > settings.max_columns_to_read) - throw Exception(ErrorCodes::TOO_MANY_COLUMNS, - "Limit for number of columns to read exceeded. Requested: {}, maximum: {}", - columns_names.size(), - settings.max_columns_to_read); - - if (columns_names.empty()) - { - auto additional_column_to_read = chooseSmallestColumnToReadFromStorage(storage, storage_snapshot); - const auto & column_identifier = planner_context->getGlobalPlannerContext()->createColumnIdentifier(additional_column_to_read, table_expression); - columns_names.push_back(additional_column_to_read.name); - table_expression_data.addColumn(additional_column_to_read, column_identifier); - } - bool need_rewrite_query_with_final = storage->needRewriteQueryWithFinal(columns_names); if (need_rewrite_query_with_final) { @@ -306,32 +451,54 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(const QueryTreeNodePtr & tabl } } - if (!select_query_options.only_analyze) - { - from_stage = storage->getQueryProcessingStage(query_context, select_query_options.to_stage, storage_snapshot, table_expression_query_info); - storage->read(query_plan, columns_names, storage_snapshot, table_expression_query_info, query_context, from_stage, max_block_size, max_streams); - } + /// Apply trivial_count optimization if possible + bool is_trivial_count_applied = !select_query_options.only_analyze && is_single_table_expression && table_node && select_query_info.has_aggregates + && applyTrivialCountIfPossible(query_plan, *table_node, select_query_info.query_tree, planner_context->getQueryContext(), columns_names); - if (query_plan.isInitialized()) + if (is_trivial_count_applied) { - /** Specify the number of threads only if it wasn't specified in storage. - * - * But in case of remote query and prefer_localhost_replica=1 (default) - * The inner local query (that is done in the same process, without - * network interaction), it will setMaxThreads earlier and distributed - * query will not update it. - */ - if (!query_plan.getMaxThreads() || is_remote) - query_plan.setMaxThreads(max_threads_execute_query); + from_stage = QueryProcessingStage::WithMergeableState; } else { - /// Create step which reads from empty source if storage has no data - auto source_header = storage_snapshot->getSampleBlockForColumns(columns_names); - Pipe pipe(std::make_shared(source_header)); - auto read_from_pipe = std::make_unique(std::move(pipe)); - read_from_pipe->setStepDescription("Read from NullSource"); - query_plan.addStep(std::move(read_from_pipe)); + if (!select_query_options.only_analyze) + { + from_stage = storage->getQueryProcessingStage(query_context, select_query_options.to_stage, storage_snapshot, table_expression_query_info); + storage->read(query_plan, columns_names, storage_snapshot, table_expression_query_info, query_context, from_stage, max_block_size, max_streams); + + if (query_context->hasQueryContext() && !select_query_options.is_internal) + { + auto local_storage_id = storage->getStorageID(); + query_context->getQueryContext()->addQueryAccessInfo( + backQuoteIfNeed(local_storage_id.getDatabaseName()), + local_storage_id.getFullTableName(), + columns_names, + {}, + {}); + } + } + + if (query_plan.isInitialized()) + { + /** Specify the number of threads only if it wasn't specified in storage. + * + * But in case of remote query and prefer_localhost_replica=1 (default) + * The inner local query (that is done in the same process, without + * network interaction), it will setMaxThreads earlier and distributed + * query will not update it. + */ + if (!query_plan.getMaxThreads() || is_remote) + query_plan.setMaxThreads(max_threads_execute_query); + } + else + { + /// Create step which reads from empty source if storage has no data. + auto source_header = storage_snapshot->getSampleBlockForColumns(columns_names); + Pipe pipe(std::make_shared(source_header)); + auto read_from_pipe = std::make_unique(std::move(pipe)); + read_from_pipe->setStepDescription("Read from NullSource"); + query_plan.addStep(std::move(read_from_pipe)); + } } } else if (query_node || union_node) @@ -350,16 +517,6 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(const QueryTreeNodePtr & tabl } else { - if (table_expression_data.getColumnNames().empty()) - { - const auto & projection_columns = query_node ? query_node->getProjectionColumns() : union_node->computeProjectionColumns(); - NamesAndTypesList projection_columns_list(projection_columns.begin(), projection_columns.end()); - auto additional_column_to_read = ExpressionActions::getSmallestColumn(projection_columns_list); - - const auto & column_identifier = planner_context->getGlobalPlannerContext()->createColumnIdentifier(additional_column_to_read, table_expression); - table_expression_data.addColumn(additional_column_to_read, column_identifier); - } - auto subquery_options = select_query_options.subquery(); Planner subquery_planner(table_expression, subquery_options, planner_context->getGlobalPlannerContext()); /// Propagate storage limits to subquery @@ -402,10 +559,11 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(const QueryTreeNodePtr & tabl planner.buildQueryPlanIfNeeded(); auto expected_header = planner.getQueryPlan().getCurrentDataStream().header; - materializeBlockInplace(expected_header); if (!blocksHaveEqualStructure(query_plan.getCurrentDataStream().header, expected_header)) { + materializeBlockInplace(expected_header); + auto rename_actions_dag = ActionsDAG::makeConvertingActions( query_plan.getCurrentDataStream().header.getColumnsWithTypeAndName(), expected_header.getColumnsWithTypeAndName(), @@ -945,14 +1103,40 @@ JoinTreeQueryPlan buildJoinTreeQueryPlan(const QueryTreeNodePtr & query_node, const ColumnIdentifierSet & outer_scope_columns, PlannerContextPtr & planner_context) { - const auto & query_node_typed = query_node->as(); - auto table_expressions_stack = buildTableExpressionsStack(query_node_typed.getJoinTree()); + auto table_expressions_stack = buildTableExpressionsStack(query_node->as().getJoinTree()); size_t table_expressions_stack_size = table_expressions_stack.size(); bool is_single_table_expression = table_expressions_stack_size == 1; std::vector table_expressions_outer_scope_columns(table_expressions_stack_size); ColumnIdentifierSet current_outer_scope_columns = outer_scope_columns; + /// For each table, table function, query, union table expressions prepare before query plan build + for (size_t i = 0; i < table_expressions_stack_size; ++i) + { + const auto & table_expression = table_expressions_stack[i]; + auto table_expression_type = table_expression->getNodeType(); + if (table_expression_type == QueryTreeNodeType::JOIN || + table_expression_type == QueryTreeNodeType::ARRAY_JOIN) + continue; + + prepareBuildQueryPlanForTableExpression(table_expression, planner_context); + } + + /** If left most table expression query plan is planned to stage that is not equal to fetch columns, + * then left most table expression is responsible for providing valid JOIN TREE part of final query plan. + * + * Examples: Distributed, LiveView, Merge storages. + */ + auto left_table_expression = table_expressions_stack.front(); + auto left_table_expression_query_plan = buildQueryPlanForTableExpression(left_table_expression, + select_query_info, + select_query_options, + planner_context, + is_single_table_expression, + false /*wrap_read_columns_in_subquery*/); + if (left_table_expression_query_plan.from_stage != QueryProcessingStage::FetchColumns) + return left_table_expression_query_plan; + for (Int64 i = static_cast(table_expressions_stack_size) - 1; i >= 0; --i) { table_expressions_outer_scope_columns[i] = current_outer_scope_columns; @@ -1006,19 +1190,23 @@ JoinTreeQueryPlan buildJoinTreeQueryPlan(const QueryTreeNodePtr & query_node, } else { - const auto & table_expression_data = planner_context->getTableExpressionDataOrThrow(table_expression); - if (table_expression_data.isRemote() && i != 0) - throw Exception(ErrorCodes::UNSUPPORTED_METHOD, - "JOIN with multiple remote storages is unsupported"); + if (table_expression == left_table_expression) + { + query_plans_stack.push_back(std::move(left_table_expression_query_plan)); /// NOLINT + left_table_expression = {}; + continue; + } + /** If table expression is remote and it is not left most table expression, we wrap read columns from such + * table expression in subquery. + */ + bool is_remote = planner_context->getTableExpressionDataOrThrow(table_expression).isRemote(); query_plans_stack.push_back(buildQueryPlanForTableExpression(table_expression, select_query_info, select_query_options, planner_context, - is_single_table_expression)); - - if (query_plans_stack.back().from_stage != QueryProcessingStage::FetchColumns) - break; + is_single_table_expression, + is_remote /*wrap_read_columns_in_subquery*/)); } } diff --git a/src/Planner/PlannerJoins.cpp b/src/Planner/PlannerJoins.cpp index e1c137ddfb8..63fe3cc7b55 100644 --- a/src/Planner/PlannerJoins.cpp +++ b/src/Planner/PlannerJoins.cpp @@ -18,6 +18,7 @@ #include #include +#include #include #include #include @@ -61,6 +62,8 @@ void JoinClause::dump(WriteBuffer & buffer) const for (const auto & dag_node : dag_nodes) { dag_nodes_dump += dag_node->result_name; + dag_nodes_dump += " "; + dag_nodes_dump += dag_node->result_type->getName(); dag_nodes_dump += ", "; } @@ -655,7 +658,7 @@ std::shared_ptr chooseJoinAlgorithm(std::shared_ptr & table_jo return std::make_shared(table_join, right_table_expression_header); } - if (!table_join->oneDisjunct() && !table_join->isEnabledAlgorithm(JoinAlgorithm::HASH)) + if (!table_join->oneDisjunct() && !table_join->isEnabledAlgorithm(JoinAlgorithm::HASH) && !table_join->isEnabledAlgorithm(JoinAlgorithm::AUTO)) throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Only `hash` join supports multiple ORs for keys in JOIN ON section"); /// Direct JOIN with special storages that support key value access. For example JOIN with Dictionary @@ -708,7 +711,11 @@ std::shared_ptr chooseJoinAlgorithm(std::shared_ptr & table_jo } if (table_join->isEnabledAlgorithm(JoinAlgorithm::AUTO)) - return std::make_shared(table_join, right_table_expression_header); + { + if (MergeJoin::isSupported(table_join)) + return std::make_shared(table_join, right_table_expression_header); + return std::make_shared(table_join, right_table_expression_header); + } throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Can't execute any of specified algorithms for specified strictness/kind and right storage type"); diff --git a/src/Planner/TableExpressionData.h b/src/Planner/TableExpressionData.h index e828f128e38..0f74e671ac7 100644 --- a/src/Planner/TableExpressionData.h +++ b/src/Planner/TableExpressionData.h @@ -101,6 +101,17 @@ public: return column_names; } + NamesAndTypes getColumns() const + { + NamesAndTypes result; + result.reserve(column_names.size()); + + for (const auto & column_name : column_names) + result.push_back(column_name_to_column.at(column_name)); + + return result; + } + ColumnIdentifiers getColumnIdentifiers() const { ColumnIdentifiers result; diff --git a/src/Planner/Utils.cpp b/src/Planner/Utils.cpp index 5c5eadac55d..2018ddafcdd 100644 --- a/src/Planner/Utils.cpp +++ b/src/Planner/Utils.cpp @@ -4,6 +4,7 @@ #include #include +#include #include #include @@ -19,6 +20,7 @@ #include #include +#include #include #include #include @@ -341,27 +343,6 @@ QueryTreeNodePtr mergeConditionNodes(const QueryTreeNodes & condition_nodes, con return function_node; } -std::optional tryExtractConstantFromConditionNode(const QueryTreeNodePtr & condition_node) -{ - const auto * constant_node = condition_node->as(); - if (!constant_node) - return {}; - - const auto & value = constant_node->getValue(); - auto constant_type = constant_node->getResultType(); - constant_type = removeNullable(removeLowCardinality(constant_type)); - - auto which_constant_type = WhichDataType(constant_type); - if (!which_constant_type.isUInt8() && !which_constant_type.isNothing()) - return {}; - - if (value.isNull()) - return false; - - UInt8 predicate_value = value.safeGet(); - return predicate_value > 0; -} - QueryTreeNodePtr replaceTablesAndTableFunctionsWithDummyTables(const QueryTreeNodePtr & query_node, const ContextPtr & context, ResultReplacementMap * result_replacement_map) @@ -391,4 +372,36 @@ QueryTreeNodePtr replaceTablesAndTableFunctionsWithDummyTables(const QueryTreeNo return query_node->cloneAndReplace(replacement_map); } +QueryTreeNodePtr buildSubqueryToReadColumnsFromTableExpression(const NamesAndTypes & columns, + const QueryTreeNodePtr & table_expression, + const ContextPtr & context) +{ + auto projection_columns = columns; + + QueryTreeNodes subquery_projection_nodes; + subquery_projection_nodes.reserve(projection_columns.size()); + + for (const auto & column : projection_columns) + subquery_projection_nodes.push_back(std::make_shared(column, table_expression)); + + if (subquery_projection_nodes.empty()) + { + auto constant_data_type = std::make_shared(); + subquery_projection_nodes.push_back(std::make_shared(1UL, constant_data_type)); + projection_columns.push_back({"1", std::move(constant_data_type)}); + } + + auto context_copy = Context::createCopy(context); + updateContextForSubqueryExecution(context_copy); + + auto query_node = std::make_shared(std::move(context_copy)); + + query_node->resolveProjectionColumns(projection_columns); + query_node->getProjection().getNodes() = std::move(subquery_projection_nodes); + query_node->getJoinTree() = table_expression; + query_node->setIsSubquery(true); + + return query_node; +} + } diff --git a/src/Planner/Utils.h b/src/Planner/Utils.h index 0520bd67d26..0effb1d08ae 100644 --- a/src/Planner/Utils.h +++ b/src/Planner/Utils.h @@ -63,13 +63,15 @@ bool queryHasWithTotalsInAnySubqueryInJoinTree(const QueryTreeNodePtr & query_no /// Returns `and` function node that has condition nodes as its arguments QueryTreeNodePtr mergeConditionNodes(const QueryTreeNodes & condition_nodes, const ContextPtr & context); -/// Try extract boolean constant from condition node -std::optional tryExtractConstantFromConditionNode(const QueryTreeNodePtr & condition_node); - /// Replace tables nodes and table function nodes with dummy table nodes using ResultReplacementMap = std::unordered_map; QueryTreeNodePtr replaceTablesAndTableFunctionsWithDummyTables(const QueryTreeNodePtr & query_node, const ContextPtr & context, ResultReplacementMap * result_replacement_map = nullptr); +/// Build subquery to read specified columns from table expression +QueryTreeNodePtr buildSubqueryToReadColumnsFromTableExpression(const NamesAndTypes & columns, + const QueryTreeNodePtr & table_expression, + const ContextPtr & context); + } diff --git a/src/Processors/Executors/PushingAsyncPipelineExecutor.cpp b/src/Processors/Executors/PushingAsyncPipelineExecutor.cpp index 54c1e7bf30f..4478f1548a4 100644 --- a/src/Processors/Executors/PushingAsyncPipelineExecutor.cpp +++ b/src/Processors/Executors/PushingAsyncPipelineExecutor.cpp @@ -139,9 +139,11 @@ PushingAsyncPipelineExecutor::PushingAsyncPipelineExecutor(QueryPipeline & pipel PushingAsyncPipelineExecutor::~PushingAsyncPipelineExecutor() { + /// It must be finalized explicitly. Otherwise we cancel it assuming it's due to an exception. + chassert(finished || std::uncaught_exceptions() || std::current_exception()); try { - finish(); + cancel(); } catch (...) { @@ -185,7 +187,7 @@ void PushingAsyncPipelineExecutor::push(Chunk chunk) if (!is_pushed) throw Exception(ErrorCodes::LOGICAL_ERROR, - "Pipeline for PushingPipelineExecutor was finished before all data was inserted"); + "Pipeline for PushingAsyncPipelineExecutor was finished before all data was inserted"); } void PushingAsyncPipelineExecutor::push(Block block) diff --git a/src/Processors/Executors/PushingPipelineExecutor.cpp b/src/Processors/Executors/PushingPipelineExecutor.cpp index d9a14704cd0..696932932df 100644 --- a/src/Processors/Executors/PushingPipelineExecutor.cpp +++ b/src/Processors/Executors/PushingPipelineExecutor.cpp @@ -63,9 +63,11 @@ PushingPipelineExecutor::PushingPipelineExecutor(QueryPipeline & pipeline_) : pi PushingPipelineExecutor::~PushingPipelineExecutor() { + /// It must be finalized explicitly. Otherwise we cancel it assuming it's due to an exception. + chassert(finished || std::uncaught_exceptions() || std::current_exception()); try { - finish(); + cancel(); } catch (...) { diff --git a/src/Processors/Formats/ISchemaReader.h b/src/Processors/Formats/ISchemaReader.h index 81bc94afa6c..e6982ea743b 100644 --- a/src/Processors/Formats/ISchemaReader.h +++ b/src/Processors/Formats/ISchemaReader.h @@ -173,19 +173,19 @@ void chooseResultColumnType( ErrorCodes::TYPE_MISMATCH, "Automatically defined type {} for column '{}' in row {} differs from type defined by previous rows: {}. " "You can specify the type for this column using setting schema_inference_hints", - type->getName(), + new_type->getName(), column_name, row, - new_type->getName()); + type->getName()); else throw Exception( ErrorCodes::TYPE_MISMATCH, "Automatically defined type {} for column '{}' in row {} differs from type defined by previous rows: {}. " "Column types from setting schema_inference_hints couldn't be parsed because of error: {}", - type->getName(), + new_type->getName(), column_name, row, - new_type->getName(), + type->getName(), hints_parsing_error); } } diff --git a/src/Processors/Formats/Impl/ArrowBlockOutputFormat.cpp b/src/Processors/Formats/Impl/ArrowBlockOutputFormat.cpp index bf0e2448082..c85c0342c8c 100644 --- a/src/Processors/Formats/Impl/ArrowBlockOutputFormat.cpp +++ b/src/Processors/Formats/Impl/ArrowBlockOutputFormat.cpp @@ -17,6 +17,24 @@ namespace ErrorCodes extern const int UNKNOWN_EXCEPTION; } +namespace +{ + +arrow::Compression::type getArrowCompression(FormatSettings::ArrowCompression method) +{ + switch (method) + { + case FormatSettings::ArrowCompression::NONE: + return arrow::Compression::type::UNCOMPRESSED; + case FormatSettings::ArrowCompression::ZSTD: + return arrow::Compression::type::ZSTD; + case FormatSettings::ArrowCompression::LZ4_FRAME: + return arrow::Compression::type::LZ4_FRAME; + } +} + +} + ArrowBlockOutputFormat::ArrowBlockOutputFormat(WriteBuffer & out_, const Block & header_, bool stream_, const FormatSettings & format_settings_) : IOutputFormat(header_, out_) , stream{stream_} @@ -78,12 +96,14 @@ void ArrowBlockOutputFormat::prepareWriter(const std::shared_ptr { arrow_ostream = std::make_shared(out); arrow::Result> writer_status; + arrow::ipc::IpcWriteOptions options = arrow::ipc::IpcWriteOptions::Defaults(); + options.codec = *arrow::util::Codec::Create(getArrowCompression(format_settings.arrow.output_compression_method)); // TODO: should we use arrow::ipc::IpcOptions::alignment? if (stream) - writer_status = arrow::ipc::MakeStreamWriter(arrow_ostream.get(), schema); + writer_status = arrow::ipc::MakeStreamWriter(arrow_ostream.get(), schema, options); else - writer_status = arrow::ipc::MakeFileWriter(arrow_ostream.get(), schema); + writer_status = arrow::ipc::MakeFileWriter(arrow_ostream.get(), schema,options); if (!writer_status.ok()) throw Exception(ErrorCodes::UNKNOWN_EXCEPTION, diff --git a/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp b/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp index f73846f15e6..54a6c8493ea 100644 --- a/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp +++ b/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -92,7 +93,7 @@ static ColumnWithTypeAndName readColumnWithNumericData(std::shared_ptr buffer = chunk->data()->buffers[1]; - const auto * raw_data = reinterpret_cast(buffer->data()); + const auto * raw_data = reinterpret_cast(buffer->data()) + chunk->offset(); column_data.insert_assume_reserved(raw_data, raw_data + chunk->length()); } return {std::move(internal_column), std::move(internal_type), column_name}; @@ -158,8 +159,8 @@ static ColumnWithTypeAndName readColumnWithFixedStringData(std::shared_ptrnum_chunks(); chunk_i < num_chunks; ++chunk_i) { arrow::FixedSizeBinaryArray & chunk = dynamic_cast(*(arrow_column->chunk(chunk_i))); - std::shared_ptr buffer = chunk.values(); - column_chars_t.insert_assume_reserved(buffer->data(), buffer->data() + buffer->size()); + const uint8_t * raw_data = chunk.raw_values(); + column_chars_t.insert_assume_reserved(raw_data, raw_data + fixed_len * chunk.length()); } return {std::move(internal_column), std::move(internal_type), column_name}; } @@ -177,9 +178,6 @@ static ColumnWithTypeAndName readColumnWithBooleanData(std::shared_ptr buffer = chunk.data()->buffers[1]; - for (size_t bool_i = 0; bool_i != static_cast(chunk.length()); ++bool_i) column_data.emplace_back(chunk.Value(bool_i)); } @@ -401,7 +399,7 @@ static ColumnWithTypeAndName readColumnWithIndexesDataImpl(std::shared_ptr buffer = chunk->data()->buffers[1]; - const auto * data = reinterpret_cast(buffer->data()); + const auto * data = reinterpret_cast(buffer->data()) + chunk->offset(); /// Check that indexes are correct (protection against corrupted files) /// Note that on null values index can be arbitrary value. @@ -528,6 +526,37 @@ static std::shared_ptr getNestedArrowColumn(std::shared_ptr return std::make_shared(array_vector); } +static ColumnWithTypeAndName readIPv6ColumnFromBinaryData(std::shared_ptr & arrow_column, const String & column_name) +{ + size_t total_size = 0; + for (int chunk_i = 0, num_chunks = arrow_column->num_chunks(); chunk_i < num_chunks; ++chunk_i) + { + auto & chunk = dynamic_cast(*(arrow_column->chunk(chunk_i))); + const size_t chunk_length = chunk.length(); + + for (size_t i = 0; i != chunk_length; ++i) + { + /// If at least one value size is not 16 bytes, fallback to reading String column and further cast to IPv6. + if (chunk.value_length(i) != sizeof(IPv6)) + return readColumnWithStringData(arrow_column, column_name); + } + total_size += chunk_length; + } + + auto internal_type = std::make_shared(); + auto internal_column = internal_type->createColumn(); + auto & data = assert_cast(*internal_column).getData(); + data.reserve(total_size * sizeof(IPv6)); + + for (int chunk_i = 0, num_chunks = arrow_column->num_chunks(); chunk_i < num_chunks; ++chunk_i) + { + auto & chunk = dynamic_cast(*(arrow_column->chunk(chunk_i))); + const auto * raw_data = reinterpret_cast(chunk.raw_data() + chunk.raw_value_offsets()[0]); + data.insert_assume_reserved(raw_data, raw_data + chunk.length()); + } + return {std::move(internal_column), std::move(internal_type), column_name}; +} + static ColumnWithTypeAndName readColumnFromArrowColumn( std::shared_ptr & arrow_column, const std::string & column_name, @@ -559,7 +588,11 @@ static ColumnWithTypeAndName readColumnFromArrowColumn( { case arrow::Type::STRING: case arrow::Type::BINARY: + { + if (type_hint && isIPv6(type_hint)) + return readIPv6ColumnFromBinaryData(arrow_column, column_name); return readColumnWithStringData(arrow_column, column_name); + } case arrow::Type::FIXED_SIZE_BINARY: return readColumnWithFixedStringData(arrow_column, column_name); case arrow::Type::LARGE_BINARY: diff --git a/src/Processors/Formats/Impl/AvroRowInputFormat.cpp b/src/Processors/Formats/Impl/AvroRowInputFormat.cpp index c3ea1b5e23b..cb851c4a1e9 100644 --- a/src/Processors/Formats/Impl/AvroRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/AvroRowInputFormat.cpp @@ -145,6 +145,9 @@ static void insertNumber(IColumn & column, WhichDataType type, T value) case TypeIndex::DateTime64: assert_cast &>(column).insertValue(static_cast(value)); break; + case TypeIndex::IPv4: + assert_cast(column).insertValue(IPv4(static_cast(value))); + break; default: throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Type is not compatible with Avro"); } @@ -424,6 +427,15 @@ AvroDeserializer::DeserializeFn AvroDeserializer::createDeserializeFn(avro::Node return true; }; } + else if (target.isIPv6() && fixed_size == sizeof(IPv6)) + { + return [tmp_fixed = std::vector(fixed_size)](IColumn & column, avro::Decoder & decoder) mutable + { + decoder.decodeFixed(tmp_fixed.size(), tmp_fixed); + column.insertData(reinterpret_cast(tmp_fixed.data()), tmp_fixed.size()); + return true; + }; + } break; } case avro::AVRO_SYMBOLIC: diff --git a/src/Processors/Formats/Impl/AvroRowOutputFormat.cpp b/src/Processors/Formats/Impl/AvroRowOutputFormat.cpp index 8483a91df62..1ca56a1c5cc 100644 --- a/src/Processors/Formats/Impl/AvroRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/AvroRowOutputFormat.cpp @@ -127,6 +127,11 @@ AvroSerializer::SchemaWithSerializeFn AvroSerializer::createSchemaWithSerializeF { encoder.encodeInt(assert_cast(column).getElement(row_num)); }}; + case TypeIndex::IPv4: + return {avro::IntSchema(), [](const IColumn & column, size_t row_num, avro::Encoder & encoder) + { + encoder.encodeInt(assert_cast(column).getElement(row_num)); + }}; case TypeIndex::Int32: return {avro::IntSchema(), [](const IColumn & column, size_t row_num, avro::Encoder & encoder) { @@ -205,6 +210,15 @@ AvroSerializer::SchemaWithSerializeFn AvroSerializer::createSchemaWithSerializeF encoder.encodeFixed(reinterpret_cast(s.data()), s.size()); }}; } + case TypeIndex::IPv6: + { + auto schema = avro::FixedSchema(sizeof(IPv6), "ipv6_" + toString(type_name_increment)); + return {schema, [](const IColumn & column, size_t row_num, avro::Encoder & encoder) + { + const std::string_view & s = assert_cast(column).getDataAt(row_num).toView(); + encoder.encodeFixed(reinterpret_cast(s.data()), s.size()); + }}; + } case TypeIndex::Enum8: { auto schema = avro::EnumSchema("enum8_" + toString(type_name_increment)); /// type names must be different for different types. diff --git a/src/Processors/Formats/Impl/BSONEachRowRowInputFormat.cpp b/src/Processors/Formats/Impl/BSONEachRowRowInputFormat.cpp index 5d747888b92..02fe58094ae 100644 --- a/src/Processors/Formats/Impl/BSONEachRowRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/BSONEachRowRowInputFormat.cpp @@ -151,6 +151,17 @@ static void readAndInsertInteger(ReadBuffer & in, IColumn & column, const DataTy } } +static void readAndInsertIPv4(ReadBuffer & in, IColumn & column, BSONType bson_type) +{ + /// We expect BSON type Int32 as IPv4 value. + if (bson_type != BSONType::INT32) + throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Cannot insert BSON Int32 into column with type IPv4"); + + UInt32 value; + readBinary(value, in); + assert_cast(column).insertValue(IPv4(value)); +} + template static void readAndInsertDouble(ReadBuffer & in, IColumn & column, const DataTypePtr & data_type, BSONType bson_type) { @@ -296,37 +307,52 @@ static void readAndInsertString(ReadBuffer & in, IColumn & column, BSONType bson } } +static void readAndInsertIPv6(ReadBuffer & in, IColumn & column, BSONType bson_type) +{ + if (bson_type != BSONType::BINARY) + throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Cannot insert BSON {} into IPv6 column", getBSONTypeName(bson_type)); + + auto size = readBSONSize(in); + auto subtype = getBSONBinarySubtype(readBSONType(in)); + if (subtype != BSONBinarySubtype::BINARY) + throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Cannot insert BSON Binary subtype {} into IPv6 column", getBSONBinarySubtypeName(subtype)); + + if (size != sizeof(IPv6)) + throw Exception( + ErrorCodes::INCORRECT_DATA, + "Cannot parse value of type IPv6, size of binary data is not equal to the binary size of IPv6 value: {} != {}", + size, + sizeof(IPv6)); + + IPv6 value; + readBinary(value, in); + assert_cast(column).insertValue(value); +} + + static void readAndInsertUUID(ReadBuffer & in, IColumn & column, BSONType bson_type) { - if (bson_type == BSONType::BINARY) - { - auto size = readBSONSize(in); - auto subtype = getBSONBinarySubtype(readBSONType(in)); - if (subtype == BSONBinarySubtype::UUID || subtype == BSONBinarySubtype::UUID_OLD) - { - if (size != sizeof(UUID)) - throw Exception( - ErrorCodes::INCORRECT_DATA, - "Cannot parse value of type UUID, size of binary data is not equal to the binary size of UUID value: {} != {}", - size, - sizeof(UUID)); - - UUID value; - readBinary(value, in); - assert_cast(column).insertValue(value); - } - else - { - throw Exception( - ErrorCodes::ILLEGAL_COLUMN, - "Cannot insert BSON Binary subtype {} into UUID column", - getBSONBinarySubtypeName(subtype)); - } - } - else - { + if (bson_type != BSONType::BINARY) throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Cannot insert BSON {} into UUID column", getBSONTypeName(bson_type)); - } + + auto size = readBSONSize(in); + auto subtype = getBSONBinarySubtype(readBSONType(in)); + if (subtype != BSONBinarySubtype::UUID && subtype != BSONBinarySubtype::UUID_OLD) + throw Exception( + ErrorCodes::ILLEGAL_COLUMN, + "Cannot insert BSON Binary subtype {} into UUID column", + getBSONBinarySubtypeName(subtype)); + + if (size != sizeof(UUID)) + throw Exception( + ErrorCodes::INCORRECT_DATA, + "Cannot parse value of type UUID, size of binary data is not equal to the binary size of UUID value: {} != {}", + size, + sizeof(UUID)); + + UUID value; + readBinary(value, in); + assert_cast(column).insertValue(value); } void BSONEachRowRowInputFormat::readArray(IColumn & column, const DataTypePtr & data_type, BSONType bson_type) @@ -591,6 +617,16 @@ bool BSONEachRowRowInputFormat::readField(IColumn & column, const DataTypePtr & readAndInsertString(*in, column, bson_type); return true; } + case TypeIndex::IPv4: + { + readAndInsertIPv4(*in, column, bson_type); + return true; + } + case TypeIndex::IPv6: + { + readAndInsertIPv6(*in, column, bson_type); + return true; + } case TypeIndex::UUID: { readAndInsertUUID(*in, column, bson_type); diff --git a/src/Processors/Formats/Impl/BSONEachRowRowOutputFormat.cpp b/src/Processors/Formats/Impl/BSONEachRowRowOutputFormat.cpp index 211021b0d78..95dd3079687 100644 --- a/src/Processors/Formats/Impl/BSONEachRowRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/BSONEachRowRowOutputFormat.cpp @@ -124,6 +124,7 @@ size_t BSONEachRowRowOutputFormat::countBSONFieldSize(const IColumn & column, co case TypeIndex::Date: [[fallthrough]]; case TypeIndex::Date32: [[fallthrough]]; case TypeIndex::Decimal32: [[fallthrough]]; + case TypeIndex::IPv4: [[fallthrough]]; case TypeIndex::Int32: { return size + sizeof(Int32); @@ -168,6 +169,10 @@ size_t BSONEachRowRowOutputFormat::countBSONFieldSize(const IColumn & column, co const auto & string_column = assert_cast(column); return size + sizeof(BSONSizeT) + string_column.getN() + 1; // Size of data + data + \0 or BSON subtype (in case of BSON binary) } + case TypeIndex::IPv6: + { + return size + sizeof(BSONSizeT) + 1 + sizeof(IPv6); // Size of data + BSON binary subtype + 16 bytes of value + } case TypeIndex::UUID: { return size + sizeof(BSONSizeT) + 1 + sizeof(UUID); // Size of data + BSON binary subtype + 16 bytes of value @@ -371,6 +376,19 @@ void BSONEachRowRowOutputFormat::serializeField(const IColumn & column, const Da writeBSONString(column, row_num, name, out, settings.bson.output_string_as_string); break; } + case TypeIndex::IPv4: + { + writeBSONNumber(BSONType::INT32, column, row_num, name, out); + break; + } + case TypeIndex::IPv6: + { + writeBSONTypeAndKeyName(BSONType::BINARY, name, out); + writeBSONSize(sizeof(IPv6), out); + writeBSONType(BSONBinarySubtype::BINARY, out); + writeBinary(assert_cast(column).getElement(row_num), out); + break; + } case TypeIndex::UUID: { writeBSONTypeAndKeyName(BSONType::BINARY, name, out); diff --git a/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp b/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp index aef54516627..8698b343eb3 100644 --- a/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp +++ b/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp @@ -434,6 +434,46 @@ namespace DB checkStatus(status, write_column->getName(), format_name); } + static void fillArrowArrayWithIPv6ColumnData( + ColumnPtr write_column, + const PaddedPODArray * null_bytemap, + const String & format_name, + arrow::ArrayBuilder* array_builder, + size_t start, + size_t end) + { + const auto & internal_column = assert_cast(*write_column); + const auto & internal_data = internal_column.getData(); + size_t fixed_length = sizeof(IPv6); + arrow::FixedSizeBinaryBuilder & builder = assert_cast(*array_builder); + arrow::Status status; + + PaddedPODArray arrow_null_bytemap = revertNullByteMap(null_bytemap, start, end); + const UInt8 * arrow_null_bytemap_raw_ptr = arrow_null_bytemap.empty() ? nullptr : arrow_null_bytemap.data(); + + const uint8_t * data_start = reinterpret_cast(internal_data.data()) + start * fixed_length; + status = builder.AppendValues(data_start, end - start, reinterpret_cast(arrow_null_bytemap_raw_ptr)); + checkStatus(status, write_column->getName(), format_name); + } + + static void fillArrowArrayWithIPv4ColumnData( + ColumnPtr write_column, + const PaddedPODArray * null_bytemap, + const String & format_name, + arrow::ArrayBuilder* array_builder, + size_t start, + size_t end) + { + const auto & internal_data = assert_cast(*write_column).getData(); + auto & builder = assert_cast(*array_builder); + arrow::Status status; + + PaddedPODArray arrow_null_bytemap = revertNullByteMap(null_bytemap, start, end); + const UInt8 * arrow_null_bytemap_raw_ptr = arrow_null_bytemap.empty() ? nullptr : arrow_null_bytemap.data(); + status = builder.AppendValues(&(internal_data.data() + start)->toUnderType(), end - start, reinterpret_cast(arrow_null_bytemap_raw_ptr)); + checkStatus(status, write_column->getName(), format_name); + } + static void fillArrowArrayWithDateColumnData( ColumnPtr write_column, const PaddedPODArray * null_bytemap, @@ -541,6 +581,14 @@ namespace DB else fillArrowArrayWithStringColumnData(column, null_bytemap, format_name, array_builder, start, end); } + else if (isIPv6(column_type)) + { + fillArrowArrayWithIPv6ColumnData(column, null_bytemap, format_name, array_builder, start, end); + } + else if (isIPv4(column_type)) + { + fillArrowArrayWithIPv4ColumnData(column, null_bytemap, format_name, array_builder, start, end); + } else if (isDate(column_type)) { fillArrowArrayWithDateColumnData(column, null_bytemap, format_name, array_builder, start, end); @@ -781,6 +829,12 @@ namespace DB if (isBool(column_type)) return arrow::boolean(); + if (isIPv6(column_type)) + return arrow::fixed_size_binary(sizeof(IPv6)); + + if (isIPv4(column_type)) + return arrow::uint32(); + const std::string type_name = column_type->getFamilyName(); if (const auto * arrow_type_it = std::find_if( internal_type_to_arrow_type.begin(), diff --git a/src/Processors/Formats/Impl/CapnProtoRowInputFormat.cpp b/src/Processors/Formats/Impl/CapnProtoRowInputFormat.cpp index 58ace9cfca5..9f37bcc3219 100644 --- a/src/Processors/Formats/Impl/CapnProtoRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/CapnProtoRowInputFormat.cpp @@ -128,6 +128,9 @@ static void insertUnsignedInteger(IColumn & column, const DataTypePtr & column_t case TypeIndex::UInt64: assert_cast(column).insertValue(value); break; + case TypeIndex::IPv4: + assert_cast(column).insertValue(IPv4(static_cast(value))); + break; default: throw Exception(ErrorCodes::LOGICAL_ERROR, "Column type is not an unsigned integer."); } diff --git a/src/Processors/Formats/Impl/CapnProtoRowOutputFormat.cpp b/src/Processors/Formats/Impl/CapnProtoRowOutputFormat.cpp index dcbd5db5f9b..c0f61bbd586 100644 --- a/src/Processors/Formats/Impl/CapnProtoRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/CapnProtoRowOutputFormat.cpp @@ -111,7 +111,12 @@ static std::optional convertToDynamicValue( case capnp::DynamicValue::Type::INT: return capnp::DynamicValue::Reader(column->getInt(row_num)); case capnp::DynamicValue::Type::UINT: + { + /// IPv4 column doesn't support getUInt method. + if (isIPv4(data_type)) + return capnp::DynamicValue::Reader(assert_cast(column.get())->getElement(row_num)); return capnp::DynamicValue::Reader(column->getUInt(row_num)); + } case capnp::DynamicValue::Type::BOOL: return capnp::DynamicValue::Reader(column->getBool(row_num)); case capnp::DynamicValue::Type::FLOAT: diff --git a/src/Processors/Formats/Impl/MsgPackRowInputFormat.cpp b/src/Processors/Formats/Impl/MsgPackRowInputFormat.cpp index 488f4ff9a73..bc41b512f79 100644 --- a/src/Processors/Formats/Impl/MsgPackRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/MsgPackRowInputFormat.cpp @@ -162,6 +162,11 @@ static void insertInteger(IColumn & column, DataTypePtr type, UInt64 value) assert_cast(column).insertValue(value); break; } + case TypeIndex::IPv4: + { + assert_cast(column).insertValue(IPv4(static_cast(value))); + break; + } default: throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Cannot insert MessagePack integer into column with type {}.", type->getName()); } @@ -190,6 +195,12 @@ static void insertString(IColumn & column, DataTypePtr type, const char * value, return; } + if (isIPv6(type) && bin) + { + assert_cast(column).insertData(value, size); + return; + } + if (!isStringOrFixedString(type)) throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Cannot insert MessagePack string into column with type {}.", type->getName()); diff --git a/src/Processors/Formats/Impl/MsgPackRowOutputFormat.cpp b/src/Processors/Formats/Impl/MsgPackRowOutputFormat.cpp index a1ed45ec40f..07951d42bc6 100644 --- a/src/Processors/Formats/Impl/MsgPackRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/MsgPackRowOutputFormat.cpp @@ -56,6 +56,11 @@ void MsgPackRowOutputFormat::serializeField(const IColumn & column, DataTypePtr packer.pack_uint32(assert_cast(column).getElement(row_num)); return; } + case TypeIndex::IPv4: + { + packer.pack_uint32(assert_cast(column).getElement(row_num)); + return; + } case TypeIndex::UInt64: { packer.pack_uint64(assert_cast(column).getElement(row_num)); @@ -110,6 +115,13 @@ void MsgPackRowOutputFormat::serializeField(const IColumn & column, DataTypePtr packer.pack_bin_body(string.data(), static_cast(string.size())); return; } + case TypeIndex::IPv6: + { + const std::string_view & data = assert_cast(column).getDataAt(row_num).toView(); + packer.pack_bin(static_cast(data.size())); + packer.pack_bin_body(data.data(), static_cast(data.size())); + return; + } case TypeIndex::Array: { auto nested_type = assert_cast(*data_type).getNestedType(); diff --git a/src/Processors/Formats/Impl/NativeFormat.cpp b/src/Processors/Formats/Impl/NativeFormat.cpp index bd1b13ce2ef..d3fd9ef73e1 100644 --- a/src/Processors/Formats/Impl/NativeFormat.cpp +++ b/src/Processors/Formats/Impl/NativeFormat.cpp @@ -23,6 +23,7 @@ public: 0, settings.skip_unknown_fields, settings.null_as_default, + settings.native.allow_types_conversion, settings.defaults_for_omitted_fields ? &block_missing_values : nullptr)) , header(header_) {} diff --git a/src/Processors/Formats/Impl/ORCBlockOutputFormat.cpp b/src/Processors/Formats/Impl/ORCBlockOutputFormat.cpp index 42c3e178436..67345a0dfb0 100644 --- a/src/Processors/Formats/Impl/ORCBlockOutputFormat.cpp +++ b/src/Processors/Formats/Impl/ORCBlockOutputFormat.cpp @@ -28,6 +28,34 @@ namespace DB namespace ErrorCodes { extern const int ILLEGAL_COLUMN; + extern const int NOT_IMPLEMENTED; +} + +namespace +{ + +orc::CompressionKind getORCCompression(FormatSettings::ORCCompression method) +{ + if (method == FormatSettings::ORCCompression::NONE) + return orc::CompressionKind::CompressionKind_NONE; + +#if USE_SNAPPY + if (method == FormatSettings::ORCCompression::SNAPPY) + return orc::CompressionKind::CompressionKind_SNAPPY; +#endif + + if (method == FormatSettings::ORCCompression::ZSTD) + return orc::CompressionKind::CompressionKind_ZSTD; + + if (method == FormatSettings::ORCCompression::LZ4) + return orc::CompressionKind::CompressionKind_LZ4; + + if (method == FormatSettings::ORCCompression::ZLIB) + return orc::CompressionKind::CompressionKind_ZLIB; + + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Unsupported compression method"); +} + } ORCOutputStream::ORCOutputStream(WriteBuffer & out_) : out(out_) {} @@ -75,6 +103,7 @@ std::unique_ptr ORCBlockOutputFormat::getORCType(const DataTypePtr & return orc::createPrimitiveType(orc::TypeKind::SHORT); } case TypeIndex::UInt32: [[fallthrough]]; + case TypeIndex::IPv4: [[fallthrough]]; case TypeIndex::Int32: { return orc::createPrimitiveType(orc::TypeKind::INT); @@ -109,6 +138,10 @@ std::unique_ptr ORCBlockOutputFormat::getORCType(const DataTypePtr & return orc::createPrimitiveType(orc::TypeKind::STRING); return orc::createPrimitiveType(orc::TypeKind::BINARY); } + case TypeIndex::IPv6: + { + return orc::createPrimitiveType(orc::TypeKind::BINARY); + } case TypeIndex::Nullable: { return getORCType(removeNullable(type)); @@ -309,6 +342,11 @@ void ORCBlockOutputFormat::writeColumn( writeNumbers(orc_column, column, null_bytemap, [](const UInt32 & value){ return value; }); break; } + case TypeIndex::IPv4: + { + writeNumbers(orc_column, column, null_bytemap, [](const IPv4 & value){ return value.toUnderType(); }); + break; + } case TypeIndex::Int64: { writeNumbers(orc_column, column, null_bytemap, [](const Int64 & value){ return value; }); @@ -339,6 +377,11 @@ void ORCBlockOutputFormat::writeColumn( writeStrings(orc_column, column, null_bytemap); break; } + case TypeIndex::IPv6: + { + writeStrings(orc_column, column, null_bytemap); + break; + } case TypeIndex::DateTime: { writeDateTimes( @@ -529,7 +572,7 @@ void ORCBlockOutputFormat::prepareWriter() { const Block & header = getPort(PortKind::Main).getHeader(); schema = orc::createStructType(); - options.setCompression(orc::CompressionKind::CompressionKind_NONE); + options.setCompression(getORCCompression(format_settings.orc.output_compression_method)); size_t columns_count = header.columns(); for (size_t i = 0; i != columns_count; ++i) schema->addStructField(header.safeGetByPosition(i).name, getORCType(recursiveRemoveLowCardinality(data_types[i]))); diff --git a/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp b/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp index fca097d8ea7..ef7575eca3c 100644 --- a/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp @@ -78,8 +78,6 @@ Chunk ParquetBlockInputFormat::generate() } else { - current_record_batch_reader.reset(); - file_reader.reset(); return {}; } @@ -91,6 +89,7 @@ void ParquetBlockInputFormat::resetParser() IInputFormat::resetParser(); file_reader.reset(); + current_record_batch_reader.reset(); column_indices.clear(); row_group_current = 0; block_missing_values.clear(); diff --git a/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp b/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp index 18c81f8fd6a..759f773a574 100644 --- a/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp +++ b/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp @@ -14,9 +14,13 @@ namespace DB namespace ErrorCodes { extern const int UNKNOWN_EXCEPTION; + extern const int NOT_IMPLEMENTED; } -static parquet::ParquetVersion::type getParquetVersion(const FormatSettings & settings) +namespace +{ + +parquet::ParquetVersion::type getParquetVersion(const FormatSettings & settings) { switch (settings.parquet.output_version) { @@ -31,6 +35,35 @@ static parquet::ParquetVersion::type getParquetVersion(const FormatSettings & se } } +parquet::Compression::type getParquetCompression(FormatSettings::ParquetCompression method) +{ + if (method == FormatSettings::ParquetCompression::NONE) + return parquet::Compression::type::UNCOMPRESSED; + +#if USE_SNAPPY + if (method == FormatSettings::ParquetCompression::SNAPPY) + return parquet::Compression::type::SNAPPY; +#endif + +#if USE_BROTLI + if (method == FormatSettings::ParquetCompression::BROTLI) + return parquet::Compression::type::BROTLI; +#endif + + if (method == FormatSettings::ParquetCompression::ZSTD) + return parquet::Compression::type::ZSTD; + + if (method == FormatSettings::ParquetCompression::LZ4) + return parquet::Compression::type::LZ4; + + if (method == FormatSettings::ParquetCompression::GZIP) + return parquet::Compression::type::GZIP; + + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Unsupported compression method"); +} + +} + ParquetBlockOutputFormat::ParquetBlockOutputFormat(WriteBuffer & out_, const Block & header_, const FormatSettings & format_settings_) : IOutputFormat(header_, out_), format_settings{format_settings_} { @@ -60,9 +93,7 @@ void ParquetBlockOutputFormat::consume(Chunk chunk) parquet::WriterProperties::Builder builder; builder.version(getParquetVersion(format_settings)); -#if USE_SNAPPY - builder.compression(parquet::Compression::SNAPPY); -#endif + builder.compression(getParquetCompression(format_settings.parquet.output_compression_method)); auto props = builder.build(); auto status = parquet::arrow::FileWriter::Open( *arrow_table->schema(), diff --git a/src/Processors/Formats/Impl/PrettySpaceBlockOutputFormat.cpp b/src/Processors/Formats/Impl/PrettySpaceBlockOutputFormat.cpp index 46d1872412c..0fb1a413a6c 100644 --- a/src/Processors/Formats/Impl/PrettySpaceBlockOutputFormat.cpp +++ b/src/Processors/Formats/Impl/PrettySpaceBlockOutputFormat.cpp @@ -45,7 +45,7 @@ void PrettySpaceBlockOutputFormat::writeChunk(const Chunk & chunk, PortKind port if (col.type->shouldAlignRightInPrettyFormats()) { - for (ssize_t k = 0; k < std::max(static_cast(0), static_cast(max_widths[i] - name_widths[i])); ++k) + for (ssize_t k = 0; k < std::max(0z, static_cast(max_widths[i] - name_widths[i])); ++k) writeChar(' ', out); if (format_settings.pretty.color) @@ -62,7 +62,7 @@ void PrettySpaceBlockOutputFormat::writeChunk(const Chunk & chunk, PortKind port if (format_settings.pretty.color) writeCString("\033[0m", out); - for (ssize_t k = 0; k < std::max(static_cast(0), static_cast(max_widths[i] - name_widths[i])); ++k) + for (ssize_t k = 0; k < std::max(0z, static_cast(max_widths[i] - name_widths[i])); ++k) writeChar(' ', out); } } diff --git a/src/Processors/Merges/Algorithms/AggregatingSortedAlgorithm.cpp b/src/Processors/Merges/Algorithms/AggregatingSortedAlgorithm.cpp index 560be60987b..ef103eb508c 100644 --- a/src/Processors/Merges/Algorithms/AggregatingSortedAlgorithm.cpp +++ b/src/Processors/Merges/Algorithms/AggregatingSortedAlgorithm.cpp @@ -117,7 +117,7 @@ static void postprocessChunk(Chunk & chunk, const AggregatingSortedAlgorithm::Co { const auto & from_type = desc.nested_type; const auto & to_type = desc.real_type; - columns[desc.column_number] = recursiveTypeConversion(columns[desc.column_number], from_type, to_type); + columns[desc.column_number] = recursiveLowCardinalityTypeConversion(columns[desc.column_number], from_type, to_type); } } diff --git a/src/Processors/Merges/Algorithms/SummingSortedAlgorithm.cpp b/src/Processors/Merges/Algorithms/SummingSortedAlgorithm.cpp index 0f1775d4ac0..d8e95e6b950 100644 --- a/src/Processors/Merges/Algorithms/SummingSortedAlgorithm.cpp +++ b/src/Processors/Merges/Algorithms/SummingSortedAlgorithm.cpp @@ -450,7 +450,7 @@ static void postprocessChunk( { const auto & from_type = desc.nested_type; const auto & to_type = desc.real_type; - res_columns[desc.column_numbers[0]] = recursiveTypeConversion(column, from_type, to_type); + res_columns[desc.column_numbers[0]] = recursiveLowCardinalityTypeConversion(column, from_type, to_type); } else res_columns[desc.column_numbers[0]] = std::move(column); diff --git a/src/Processors/QueryPlan/AggregatingStep.cpp b/src/Processors/QueryPlan/AggregatingStep.cpp index 9bf351442b2..69dfa05899b 100644 --- a/src/Processors/QueryPlan/AggregatingStep.cpp +++ b/src/Processors/QueryPlan/AggregatingStep.cpp @@ -38,7 +38,6 @@ static ITransformingStep::Traits getTraits(bool should_produce_results_in_order_ return ITransformingStep::Traits { { - .preserves_distinct_columns = false, /// Actually, we may check that distinct names are in aggregation keys .returns_single_stream = should_produce_results_in_order_of_bucket_number, .preserves_number_of_streams = false, .preserves_sorting = false, diff --git a/src/Processors/QueryPlan/ArrayJoinStep.cpp b/src/Processors/QueryPlan/ArrayJoinStep.cpp index bd1908a4a6d..23a0a756f0d 100644 --- a/src/Processors/QueryPlan/ArrayJoinStep.cpp +++ b/src/Processors/QueryPlan/ArrayJoinStep.cpp @@ -14,7 +14,6 @@ static ITransformingStep::Traits getTraits() return ITransformingStep::Traits { { - .preserves_distinct_columns = false, .returns_single_stream = false, .preserves_number_of_streams = true, .preserves_sorting = false, diff --git a/src/Processors/QueryPlan/CreateSetAndFilterOnTheFlyStep.cpp b/src/Processors/QueryPlan/CreateSetAndFilterOnTheFlyStep.cpp index 53dcec9ef0a..07137e87736 100644 --- a/src/Processors/QueryPlan/CreateSetAndFilterOnTheFlyStep.cpp +++ b/src/Processors/QueryPlan/CreateSetAndFilterOnTheFlyStep.cpp @@ -40,7 +40,6 @@ static ITransformingStep::Traits getTraits() return ITransformingStep::Traits { { - .preserves_distinct_columns = true, .returns_single_stream = false, .preserves_number_of_streams = true, .preserves_sorting = true, diff --git a/src/Processors/QueryPlan/CreatingSetsStep.cpp b/src/Processors/QueryPlan/CreatingSetsStep.cpp index 23e0a17a31b..b696b77ccfe 100644 --- a/src/Processors/QueryPlan/CreatingSetsStep.cpp +++ b/src/Processors/QueryPlan/CreatingSetsStep.cpp @@ -21,7 +21,6 @@ static ITransformingStep::Traits getTraits() return ITransformingStep::Traits { { - .preserves_distinct_columns = true, .returns_single_stream = false, .preserves_number_of_streams = true, .preserves_sorting = true, diff --git a/src/Processors/QueryPlan/CubeStep.cpp b/src/Processors/QueryPlan/CubeStep.cpp index 03f952ac782..0c632c346c7 100644 --- a/src/Processors/QueryPlan/CubeStep.cpp +++ b/src/Processors/QueryPlan/CubeStep.cpp @@ -14,7 +14,6 @@ static ITransformingStep::Traits getTraits() return ITransformingStep::Traits { { - .preserves_distinct_columns = false, .returns_single_stream = true, .preserves_number_of_streams = false, .preserves_sorting = false, @@ -32,9 +31,6 @@ CubeStep::CubeStep(const DataStream & input_stream_, Aggregator::Params params_, , final(final_) , use_nulls(use_nulls_) { - /// Aggregation keys are distinct - for (const auto & key : params.keys) - output_stream->distinct_columns.insert(key); } ProcessorPtr addGroupingSetForTotals(const Block & header, const Names & keys, bool use_nulls, const BuildQueryPipelineSettings & settings, UInt64 grouping_set_number) @@ -89,9 +85,5 @@ void CubeStep::updateOutputStream() { output_stream = createOutputStream( input_streams.front(), generateOutputHeader(params.getHeader(input_streams.front().header, final), params.keys, use_nulls), getDataStreamTraits()); - - /// Aggregation keys are distinct - for (const auto & key : params.keys) - output_stream->distinct_columns.insert(key); } } diff --git a/src/Processors/QueryPlan/DistinctStep.cpp b/src/Processors/QueryPlan/DistinctStep.cpp index 323ef0bbdab..15ed02b700e 100644 --- a/src/Processors/QueryPlan/DistinctStep.cpp +++ b/src/Processors/QueryPlan/DistinctStep.cpp @@ -10,28 +10,13 @@ namespace DB { -static bool checkColumnsAlreadyDistinct(const Names & columns, const NameSet & distinct_names) -{ - if (distinct_names.empty()) - return false; - - /// Now we need to check that distinct_names is a subset of columns. - std::unordered_set columns_set(columns.begin(), columns.end()); - for (const auto & name : distinct_names) - if (!columns_set.contains(name)) - return false; - - return true; -} - -static ITransformingStep::Traits getTraits(bool pre_distinct, bool already_distinct_columns) +static ITransformingStep::Traits getTraits(bool pre_distinct) { return ITransformingStep::Traits { { - .preserves_distinct_columns = already_distinct_columns, /// Will be calculated separately otherwise - .returns_single_stream = !pre_distinct && !already_distinct_columns, - .preserves_number_of_streams = pre_distinct || already_distinct_columns, + .returns_single_stream = !pre_distinct, + .preserves_number_of_streams = pre_distinct, .preserves_sorting = true, /// Sorting is preserved indeed because of implementation. }, { @@ -62,34 +47,23 @@ DistinctStep::DistinctStep( : ITransformingStep( input_stream_, input_stream_.header, - getTraits(pre_distinct_, checkColumnsAlreadyDistinct(columns_, input_stream_.distinct_columns))) + getTraits(pre_distinct_)) , set_size_limits(set_size_limits_) , limit_hint(limit_hint_) , columns(columns_) , pre_distinct(pre_distinct_) , optimize_distinct_in_order(optimize_distinct_in_order_) { - if (!output_stream->distinct_columns.empty() /// Columns already distinct, do nothing - && (!pre_distinct /// Main distinct - || input_stream_.has_single_port)) /// pre_distinct for single port works as usual one - { - /// Build distinct set. - for (const auto & name : columns) - output_stream->distinct_columns.insert(name); - } } void DistinctStep::transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &) { - const auto & input_stream = input_streams.back(); - if (checkColumnsAlreadyDistinct(columns, input_stream.distinct_columns)) - return; - if (!pre_distinct) pipeline.resize(1); if (optimize_distinct_in_order) { + const auto & input_stream = input_streams.back(); const SortDescription distinct_sort_desc = getSortDescription(input_stream.sort_description, columns); if (!distinct_sort_desc.empty()) { @@ -197,16 +171,7 @@ void DistinctStep::updateOutputStream() output_stream = createOutputStream( input_streams.front(), input_streams.front().header, - getTraits(pre_distinct, checkColumnsAlreadyDistinct(columns, input_streams.front().distinct_columns)).data_stream_traits); - - if (!output_stream->distinct_columns.empty() /// Columns already distinct, do nothing - && (!pre_distinct /// Main distinct - || input_streams.front().has_single_port)) /// pre_distinct for single port works as usual one - { - /// Build distinct set. - for (const auto & name : columns) - output_stream->distinct_columns.insert(name); - } + getTraits(pre_distinct).data_stream_traits); } } diff --git a/src/Processors/QueryPlan/ExpressionStep.cpp b/src/Processors/QueryPlan/ExpressionStep.cpp index dcfa6e5a891..250a1733caa 100644 --- a/src/Processors/QueryPlan/ExpressionStep.cpp +++ b/src/Processors/QueryPlan/ExpressionStep.cpp @@ -15,7 +15,6 @@ static ITransformingStep::Traits getTraits(const ActionsDAGPtr & actions, const return ITransformingStep::Traits { { - .preserves_distinct_columns = !actions->hasArrayJoin(), .returns_single_stream = false, .preserves_number_of_streams = true, .preserves_sorting = actions->isSortingPreserved(header, sort_description), @@ -33,8 +32,6 @@ ExpressionStep::ExpressionStep(const DataStream & input_stream_, const ActionsDA getTraits(actions_dag_, input_stream_.header, input_stream_.sort_description)) , actions_dag(actions_dag_) { - /// Some columns may be removed by expression. - updateDistinctColumns(output_stream->header, output_stream->distinct_columns); } void ExpressionStep::transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings & settings) @@ -63,22 +60,9 @@ void ExpressionStep::transformPipeline(QueryPipelineBuilder & pipeline, const Bu void ExpressionStep::describeActions(FormatSettings & settings) const { - String prefix(settings.offset, ' '); - bool first = true; - + String prefix(settings.offset, settings.indent_char); auto expression = std::make_shared(actions_dag); - for (const auto & action : expression->getActions()) - { - settings.out << prefix << (first ? "Actions: " - : " "); - first = false; - settings.out << action.toString() << '\n'; - } - - settings.out << prefix << "Positions:"; - for (const auto & pos : expression->getResultPositions()) - settings.out << ' ' << pos; - settings.out << '\n'; + expression->describeActions(settings.out, prefix); } void ExpressionStep::describeActions(JSONBuilder::JSONMap & map) const diff --git a/src/Processors/QueryPlan/ExtremesStep.cpp b/src/Processors/QueryPlan/ExtremesStep.cpp index 4524b9883d6..010a82072cf 100644 --- a/src/Processors/QueryPlan/ExtremesStep.cpp +++ b/src/Processors/QueryPlan/ExtremesStep.cpp @@ -9,7 +9,6 @@ static ITransformingStep::Traits getTraits() return ITransformingStep::Traits { { - .preserves_distinct_columns = true, .returns_single_stream = false, .preserves_number_of_streams = true, .preserves_sorting = true, diff --git a/src/Processors/QueryPlan/FillingStep.cpp b/src/Processors/QueryPlan/FillingStep.cpp index dde3bdbf850..20d7d6d0f8f 100644 --- a/src/Processors/QueryPlan/FillingStep.cpp +++ b/src/Processors/QueryPlan/FillingStep.cpp @@ -17,7 +17,6 @@ static ITransformingStep::Traits getTraits() return ITransformingStep::Traits { { - .preserves_distinct_columns = false, /// TODO: it seem to actually be true. Check it later. .returns_single_stream = true, .preserves_number_of_streams = true, .preserves_sorting = true, @@ -40,8 +39,10 @@ void FillingStep::transformPipeline(QueryPipelineBuilder & pipeline, const Build { pipeline.addSimpleTransform([&](const Block & header, QueryPipelineBuilder::StreamType stream_type) -> ProcessorPtr { - bool on_totals = stream_type == QueryPipelineBuilder::StreamType::Totals; - return std::make_shared(header, sort_description, std::move(interpolate_description), on_totals); + if (stream_type == QueryPipelineBuilder::StreamType::Totals) + return std::make_shared(header, sort_description); + + return std::make_shared(header, sort_description, std::move(interpolate_description)); }); } diff --git a/src/Processors/QueryPlan/FilterStep.cpp b/src/Processors/QueryPlan/FilterStep.cpp index 4699a7c1908..dc837446a96 100644 --- a/src/Processors/QueryPlan/FilterStep.cpp +++ b/src/Processors/QueryPlan/FilterStep.cpp @@ -23,7 +23,6 @@ static ITransformingStep::Traits getTraits(const ActionsDAGPtr & expression, con return ITransformingStep::Traits { { - .preserves_distinct_columns = !expression->hasArrayJoin(), /// I suppose it actually never happens .returns_single_stream = false, .preserves_number_of_streams = true, .preserves_sorting = preserves_sorting, @@ -51,8 +50,6 @@ FilterStep::FilterStep( , filter_column_name(std::move(filter_column_name_)) , remove_filter_column(remove_filter_column_) { - /// TODO: it would be easier to remove all expressions from filter step. It should only filter by column name. - updateDistinctColumns(output_stream->header, output_stream->distinct_columns); } void FilterStep::transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings & settings) @@ -82,27 +79,15 @@ void FilterStep::transformPipeline(QueryPipelineBuilder & pipeline, const BuildQ void FilterStep::describeActions(FormatSettings & settings) const { - String prefix(settings.offset, ' '); + String prefix(settings.offset, settings.indent_char); settings.out << prefix << "Filter column: " << filter_column_name; if (remove_filter_column) settings.out << " (removed)"; settings.out << '\n'; - bool first = true; auto expression = std::make_shared(actions_dag); - for (const auto & action : expression->getActions()) - { - settings.out << prefix << (first ? "Actions: " - : " "); - first = false; - settings.out << action.toString() << '\n'; - } - - settings.out << prefix << "Positions:"; - for (const auto & pos : expression->getResultPositions()) - settings.out << ' ' << pos; - settings.out << '\n'; + expression->describeActions(settings.out, prefix); } void FilterStep::describeActions(JSONBuilder::JSONMap & map) const diff --git a/src/Processors/QueryPlan/IQueryPlanStep.h b/src/Processors/QueryPlan/IQueryPlanStep.h index 316ecff9c2e..a608c6f8058 100644 --- a/src/Processors/QueryPlan/IQueryPlanStep.h +++ b/src/Processors/QueryPlan/IQueryPlanStep.h @@ -23,11 +23,6 @@ class DataStream public: Block header; - /// Tuples with those columns are distinct. - /// It doesn't mean that columns are distinct separately. - /// Removing any column from this list breaks this invariant. - NameSet distinct_columns = {}; - /// QueryPipeline has single port. Totals or extremes ports are not counted. bool has_single_port = false; @@ -51,8 +46,7 @@ public: bool hasEqualPropertiesWith(const DataStream & other) const { - return distinct_columns == other.distinct_columns - && has_single_port == other.has_single_port + return has_single_port == other.has_single_port && sort_description == other.sort_description && (sort_description.empty() || sort_scope == other.sort_scope); } diff --git a/src/Processors/QueryPlan/ISourceStep.cpp b/src/Processors/QueryPlan/ISourceStep.cpp index 0644d9b44eb..37f56bc7a43 100644 --- a/src/Processors/QueryPlan/ISourceStep.cpp +++ b/src/Processors/QueryPlan/ISourceStep.cpp @@ -12,10 +12,19 @@ ISourceStep::ISourceStep(DataStream output_stream_) QueryPipelineBuilderPtr ISourceStep::updatePipeline(QueryPipelineBuilders, const BuildQueryPipelineSettings & settings) { auto pipeline = std::make_unique(); - QueryPipelineProcessorsCollector collector(*pipeline, this); + + /// For `Source` step, since it's not add new Processors to `pipeline->pipe` + /// in `initializePipeline`, but make an assign with new created Pipe. + /// And Processors for the Step is added here. So we do not need to use + /// `QueryPipelineProcessorsCollector` to collect Processors. initializePipeline(*pipeline, settings); - auto added_processors = collector.detachProcessors(); - processors.insert(processors.end(), added_processors.begin(), added_processors.end()); + + /// But we need to set QueryPlanStep manually for the Processors, which + /// will be used in `EXPLAIN PIPELINE` + for (auto & processor : processors) + { + processor->setQueryPlanStep(this); + } return pipeline; } diff --git a/src/Processors/QueryPlan/ITransformingStep.cpp b/src/Processors/QueryPlan/ITransformingStep.cpp index 195fa9ad68c..9ecfdb0af22 100644 --- a/src/Processors/QueryPlan/ITransformingStep.cpp +++ b/src/Processors/QueryPlan/ITransformingStep.cpp @@ -20,9 +20,6 @@ DataStream ITransformingStep::createOutputStream( { DataStream output_stream{.header = std::move(output_header)}; - if (stream_traits.preserves_distinct_columns) - output_stream.distinct_columns = input_stream.distinct_columns; - output_stream.has_single_port = stream_traits.returns_single_stream || (input_stream.has_single_port && stream_traits.preserves_number_of_streams); @@ -50,21 +47,6 @@ QueryPipelineBuilderPtr ITransformingStep::updatePipeline(QueryPipelineBuilders return std::move(pipelines.front()); } -void ITransformingStep::updateDistinctColumns(const Block & res_header, NameSet & distinct_columns) -{ - if (distinct_columns.empty()) - return; - - for (const auto & column : distinct_columns) - { - if (!res_header.has(column)) - { - distinct_columns.clear(); - break; - } - } -} - void ITransformingStep::describePipeline(FormatSettings & settings) const { IQueryPlanStep::describePipeline(processors, settings); diff --git a/src/Processors/QueryPlan/ITransformingStep.h b/src/Processors/QueryPlan/ITransformingStep.h index 1513b4307f8..77de668fbdb 100644 --- a/src/Processors/QueryPlan/ITransformingStep.h +++ b/src/Processors/QueryPlan/ITransformingStep.h @@ -18,11 +18,6 @@ public: /// They are specified in constructor and cannot be changed. struct DataStreamTraits { - /// Keep distinct_columns unchanged. - /// Examples: true for LimitStep, false for ExpressionStep with ARRAY JOIN - /// It some columns may be removed from result header, call updateDistinctColumns - bool preserves_distinct_columns; - /// True if pipeline has single output port after this step. /// Examples: MergeSortingStep, AggregatingStep bool returns_single_stream; @@ -69,8 +64,6 @@ public: input_streams.emplace_back(std::move(input_stream)); updateOutputStream(); - - updateDistinctColumns(output_stream->header, output_stream->distinct_columns); } void describePipeline(FormatSettings & settings) const override; @@ -83,9 +76,6 @@ public: } protected: - /// Clear distinct_columns if res_header doesn't contain all of them. - static void updateDistinctColumns(const Block & res_header, NameSet & distinct_columns); - /// Create output stream from header and traits. static DataStream createOutputStream( const DataStream & input_stream, diff --git a/src/Processors/QueryPlan/JoinStep.cpp b/src/Processors/QueryPlan/JoinStep.cpp index 6e212a53bc6..2ff8f161e99 100644 --- a/src/Processors/QueryPlan/JoinStep.cpp +++ b/src/Processors/QueryPlan/JoinStep.cpp @@ -83,7 +83,6 @@ static ITransformingStep::Traits getStorageJoinTraits() return ITransformingStep::Traits { { - .preserves_distinct_columns = false, .returns_single_stream = false, .preserves_number_of_streams = true, .preserves_sorting = false, diff --git a/src/Processors/QueryPlan/JoinStep.h b/src/Processors/QueryPlan/JoinStep.h index a814d541574..e7185f36588 100644 --- a/src/Processors/QueryPlan/JoinStep.h +++ b/src/Processors/QueryPlan/JoinStep.h @@ -49,6 +49,8 @@ public: String getName() const override { return "FilledJoin"; } void transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &) override; + const JoinPtr & getJoin() const { return join; } + private: void updateOutputStream() override; diff --git a/src/Processors/QueryPlan/LimitByStep.cpp b/src/Processors/QueryPlan/LimitByStep.cpp index 39086e995fc..8b4abecc12c 100644 --- a/src/Processors/QueryPlan/LimitByStep.cpp +++ b/src/Processors/QueryPlan/LimitByStep.cpp @@ -12,7 +12,6 @@ static ITransformingStep::Traits getTraits() return ITransformingStep::Traits { { - .preserves_distinct_columns = true, .returns_single_stream = true, .preserves_number_of_streams = false, .preserves_sorting = true, diff --git a/src/Processors/QueryPlan/LimitStep.cpp b/src/Processors/QueryPlan/LimitStep.cpp index 144ac16f0d5..5e5a7387832 100644 --- a/src/Processors/QueryPlan/LimitStep.cpp +++ b/src/Processors/QueryPlan/LimitStep.cpp @@ -12,7 +12,6 @@ static ITransformingStep::Traits getTraits() return ITransformingStep::Traits { { - .preserves_distinct_columns = true, .returns_single_stream = false, .preserves_number_of_streams = true, .preserves_sorting = true, diff --git a/src/Processors/QueryPlan/MergingAggregatedStep.cpp b/src/Processors/QueryPlan/MergingAggregatedStep.cpp index e4fc332a1fd..8b5f21442b1 100644 --- a/src/Processors/QueryPlan/MergingAggregatedStep.cpp +++ b/src/Processors/QueryPlan/MergingAggregatedStep.cpp @@ -24,7 +24,6 @@ static ITransformingStep::Traits getTraits(bool should_produce_results_in_order_ return ITransformingStep::Traits { { - .preserves_distinct_columns = false, .returns_single_stream = should_produce_results_in_order_of_bucket_number, .preserves_number_of_streams = false, .preserves_sorting = false, @@ -62,10 +61,6 @@ MergingAggregatedStep::MergingAggregatedStep( , should_produce_results_in_order_of_bucket_number(should_produce_results_in_order_of_bucket_number_) , memory_bound_merging_of_aggregation_results_enabled(memory_bound_merging_of_aggregation_results_enabled_) { - /// Aggregation keys are distinct - for (const auto & key : params.keys) - output_stream->distinct_columns.insert(key); - if (memoryBoundMergingWillBeUsed() && should_produce_results_in_order_of_bucket_number) { output_stream->sort_description = group_by_sort_description; @@ -157,10 +152,6 @@ void MergingAggregatedStep::describeActions(JSONBuilder::JSONMap & map) const void MergingAggregatedStep::updateOutputStream() { output_stream = createOutputStream(input_streams.front(), params.getHeader(input_streams.front().header, final), getDataStreamTraits()); - - /// Aggregation keys are distinct - for (const auto & key : params.keys) - output_stream->distinct_columns.insert(key); } bool MergingAggregatedStep::memoryBoundMergingWillBeUsed() const diff --git a/src/Processors/QueryPlan/MergingAggregatedStep.h b/src/Processors/QueryPlan/MergingAggregatedStep.h index 2dea289ca89..3a7e2b66183 100644 --- a/src/Processors/QueryPlan/MergingAggregatedStep.h +++ b/src/Processors/QueryPlan/MergingAggregatedStep.h @@ -27,6 +27,7 @@ public: bool memory_bound_merging_of_aggregation_results_enabled_); String getName() const override { return "MergingAggregated"; } + const Aggregator::Params & getParams() const { return params; } void transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &) override; diff --git a/src/Processors/QueryPlan/OffsetStep.cpp b/src/Processors/QueryPlan/OffsetStep.cpp index e0c70ba2f28..4bbe81f9169 100644 --- a/src/Processors/QueryPlan/OffsetStep.cpp +++ b/src/Processors/QueryPlan/OffsetStep.cpp @@ -12,7 +12,6 @@ static ITransformingStep::Traits getTraits() return ITransformingStep::Traits { { - .preserves_distinct_columns = true, .returns_single_stream = false, .preserves_number_of_streams = true, .preserves_sorting = true, diff --git a/src/Processors/QueryPlan/Optimizations/distinctReadInOrder.cpp b/src/Processors/QueryPlan/Optimizations/distinctReadInOrder.cpp index d584a27f16e..6334594de30 100644 --- a/src/Processors/QueryPlan/Optimizations/distinctReadInOrder.cpp +++ b/src/Processors/QueryPlan/Optimizations/distinctReadInOrder.cpp @@ -1,5 +1,7 @@ #include #include +#include +#include #include #include #include @@ -7,6 +9,71 @@ namespace DB::QueryPlanOptimizations { +/// build actions DAG from stack of steps +static ActionsDAGPtr buildActionsForPlanPath(std::vector & dag_stack) +{ + if (dag_stack.empty()) + return nullptr; + + ActionsDAGPtr path_actions = dag_stack.back()->clone(); + dag_stack.pop_back(); + while (!dag_stack.empty()) + { + ActionsDAGPtr clone = dag_stack.back()->clone(); + dag_stack.pop_back(); + path_actions->mergeInplace(std::move(*clone)); + } + return path_actions; +} + +static const ActionsDAG::Node * getOriginalNodeForOutputAlias(const ActionsDAGPtr & actions, const String & output_name) +{ + /// find alias in output + const ActionsDAG::Node * output_alias = nullptr; + for (const auto * node : actions->getOutputs()) + { + if (node->result_name == output_name) + { + output_alias = node; + break; + } + } + if (!output_alias) + return nullptr; + + /// find original(non alias) node it refers to + const ActionsDAG::Node * node = output_alias; + while (node && node->type == ActionsDAG::ActionType::ALIAS) + { + chassert(!node->children.empty()); + node = node->children.front(); + } + if (node && node->type != ActionsDAG::ActionType::INPUT) + return nullptr; + + return node; +} + +static std::set +getOriginalDistinctColumns(const ColumnsWithTypeAndName & distinct_columns, std::vector & dag_stack) +{ + auto actions = buildActionsForPlanPath(dag_stack); + std::set original_distinct_columns; + for (const auto & column : distinct_columns) + { + /// const columns doesn't affect DISTINCT, so skip them + if (isColumnConst(*column.column)) + continue; + + const auto * input_node = getOriginalNodeForOutputAlias(actions, column.name); + if (!input_node) + break; + + original_distinct_columns.insert(input_node->result_name); + } + return original_distinct_columns; +} + size_t tryDistinctReadInOrder(QueryPlan::Node * parent_node) { /// check if it is preliminary distinct node @@ -22,8 +89,10 @@ size_t tryDistinctReadInOrder(QueryPlan::Node * parent_node) /// walk through the plan /// (1) check if nodes below preliminary distinct preserve sorting /// (2) gather transforming steps to update their sorting properties later + /// (3) gather actions DAG to find original names for columns in distinct step later std::vector steps_to_update; QueryPlan::Node * node = parent_node; + std::vector dag_stack; while (!node->children.empty()) { auto * step = dynamic_cast(node->step.get()); @@ -36,6 +105,11 @@ size_t tryDistinctReadInOrder(QueryPlan::Node * parent_node) steps_to_update.push_back(step); + if (const auto * const expr = typeid_cast(step); expr) + dag_stack.push_back(expr->getExpression()); + else if (const auto * const filter = typeid_cast(step); filter) + dag_stack.push_back(filter->getExpression()); + node = node->children.front(); } @@ -50,28 +124,24 @@ size_t tryDistinctReadInOrder(QueryPlan::Node * parent_node) if (read_from_merge_tree->getOutputStream().sort_description.empty()) return 0; - /// find non-const columns in DISTINCT + /// get original names for DISTINCT columns const ColumnsWithTypeAndName & distinct_columns = pre_distinct->getOutputStream().header.getColumnsWithTypeAndName(); - std::set non_const_columns; - for (const auto & column : distinct_columns) - { - if (!isColumnConst(*column.column)) - non_const_columns.emplace(column.name); - } + auto original_distinct_columns = getOriginalDistinctColumns(distinct_columns, dag_stack); - const Names& sorting_key_columns = read_from_merge_tree->getStorageMetadata()->getSortingKeyColumns(); /// check if DISTINCT has the same columns as sorting key + const Names & sorting_key_columns = read_from_merge_tree->getStorageMetadata()->getSortingKeyColumns(); size_t number_of_sorted_distinct_columns = 0; for (const auto & column_name : sorting_key_columns) { - if (non_const_columns.end() == non_const_columns.find(column_name)) + if (!original_distinct_columns.contains(column_name)) break; ++number_of_sorted_distinct_columns; } + /// apply optimization only when distinct columns match or form prefix of sorting key /// todo: check if reading in order optimization would be beneficial when sorting key is prefix of columns in DISTINCT - if (number_of_sorted_distinct_columns != non_const_columns.size()) + if (number_of_sorted_distinct_columns != original_distinct_columns.size()) return 0; /// check if another read in order optimization is already applied diff --git a/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp b/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp index d466c52725f..37bc894339f 100644 --- a/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp +++ b/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp @@ -314,11 +314,14 @@ size_t tryPushDownFilter(QueryPlan::Node * parent_node, QueryPlan::Nodes & nodes if (auto updated_steps = simplePushDownOverStep(parent_node, nodes, child)) return updated_steps; - if (auto * join = typeid_cast(child.get())) + auto * join = typeid_cast(child.get()); + auto * filled_join = typeid_cast(child.get()); + + if (join || filled_join) { auto join_push_down = [&](JoinKind kind) -> size_t { - const auto & table_join = join->getJoin()->getTableJoin(); + const auto & table_join = join ? join->getJoin()->getTableJoin() : filled_join->getJoin()->getTableJoin(); /// Only inner and left(/right) join are supported. Other types may generate default values for left table keys. /// So, if we push down a condition like `key != 0`, not all rows may be filtered. @@ -326,8 +329,8 @@ size_t tryPushDownFilter(QueryPlan::Node * parent_node, QueryPlan::Nodes & nodes return 0; bool is_left = kind == JoinKind::Left; - const auto & input_header = is_left ? join->getInputStreams().front().header : join->getInputStreams().back().header; - const auto & res_header = join->getOutputStream().header; + const auto & input_header = is_left ? child->getInputStreams().front().header : child->getInputStreams().back().header; + const auto & res_header = child->getOutputStream().header; Names allowed_keys; const auto & source_columns = input_header.getNames(); for (const auto & name : source_columns) @@ -372,7 +375,7 @@ size_t tryPushDownFilter(QueryPlan::Node * parent_node, QueryPlan::Nodes & nodes return updated_steps; /// For full sorting merge join we push down both to the left and right tables, because left and right streams are not independent. - if (join->allowPushDownToRight()) + if (join && join->allowPushDownToRight()) { if (size_t updated_steps = join_push_down(JoinKind::Right)) return updated_steps; diff --git a/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp b/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp index 0874a3771ae..9407504579b 100644 --- a/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp +++ b/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp @@ -519,8 +519,9 @@ AggregationInputOrder buildInputOrderInfo( enreachFixedColumns(sorting_key_dag, fixed_key_columns); - for (auto it = matches.cbegin(); it != matches.cend(); ++it) + for (const auto * output : dag->getOutputs()) { + auto it = matches.find(output); const MatchedTrees::Match * match = &it->second; if (match->node) { diff --git a/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp b/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp index 02725dc3122..c9a0270f6e7 100644 --- a/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp +++ b/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -100,24 +101,29 @@ namespace logDebug("aggregation_keys", aggregation_keys); logDebug("aggregation_keys size", aggregation_keys.size()); logDebug("distinct_columns size", distinct_columns.size()); - if (aggregation_keys.size() != distinct_columns.size()) - return false; - /// compare columns of two DISTINCTs + std::set original_distinct_columns; for (const auto & column : distinct_columns) { logDebug("distinct column name", column); const auto * alias_node = getOriginalNodeForOutputAlias(path_actions, String(column)); if (!alias_node) { - logDebug("original name for alias is not found for", column); - return false; + logDebug("original name for alias is not found", column); + original_distinct_columns.insert(column); } - - logDebug("alias result name", alias_node->result_name); - if (std::find(cbegin(aggregation_keys), cend(aggregation_keys), alias_node->result_name) == aggregation_keys.cend()) + else { - logDebug("alias result name is not found in aggregation keys", alias_node->result_name); + logDebug("alias result name", alias_node->result_name); + original_distinct_columns.insert(alias_node->result_name); + } + } + /// if aggregation keys are part of distinct columns then rows already distinct + for (const auto & key : aggregation_keys) + { + if (!original_distinct_columns.contains(key)) + { + logDebug("aggregation key NOT found: {}", key); return false; } } @@ -176,7 +182,7 @@ namespace while (!node->children.empty()) { const IQueryPlanStep * current_step = node->step.get(); - if (typeid_cast(current_step)) + if (typeid_cast(current_step) || typeid_cast(current_step)) { aggregation_before_distinct = current_step; break; @@ -208,6 +214,9 @@ namespace if (const auto * aggregating_step = typeid_cast(aggregation_before_distinct); aggregating_step) return compareAggregationKeysWithDistinctColumns(aggregating_step->getParams().keys, distinct_columns, actions); + else if (const auto * merging_aggregated_step = typeid_cast(aggregation_before_distinct); + merging_aggregated_step) + return compareAggregationKeysWithDistinctColumns(merging_aggregated_step->getParams().keys, distinct_columns, actions); } return false; diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index 51afe96545d..753bb070c47 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -340,57 +340,55 @@ Pipe ReadFromMergeTree::readFromPool( / max_block_size * max_block_size / fixed_index_granularity; } - bool all_parts_are_remote = true; - bool all_parts_are_local = true; - for (const auto & part : parts_with_range) - { - const bool is_remote = part.data_part->isStoredOnRemoteDisk(); - all_parts_are_local &= !is_remote; - all_parts_are_remote &= is_remote; - } + bool all_parts_are_remote = true; + bool all_parts_are_local = true; + for (const auto & part : parts_with_range) + { + const bool is_remote = part.data_part->isStoredOnRemoteDisk(); + all_parts_are_local &= !is_remote; + all_parts_are_remote &= is_remote; + } - MergeTreeReadPoolPtr pool; + MergeTreeReadPoolPtr pool; - if ((all_parts_are_remote - && settings.allow_prefetched_read_pool_for_remote_filesystem - && MergeTreePrefetchedReadPool::checkReadMethodAllowed(reader_settings.read_settings.remote_fs_method)) - || (all_parts_are_local - && settings.allow_prefetched_read_pool_for_local_filesystem - && MergeTreePrefetchedReadPool::checkReadMethodAllowed(reader_settings.read_settings.local_fs_method))) - { - pool = std::make_shared( - max_streams, - sum_marks, - min_marks_for_concurrent_read, - std::move(parts_with_range), - storage_snapshot, - prewhere_info, - actions_settings, - required_columns, - virt_column_names, - settings.preferred_block_size_bytes, - reader_settings, - context, - use_uncompressed_cache, - all_parts_are_remote, - *data.getSettings()); - } - else - { - pool = std::make_shared( - max_streams, - sum_marks, - min_marks_for_concurrent_read, - std::move(parts_with_range), - storage_snapshot, - prewhere_info, - actions_settings, - reader_settings, - required_columns, - virt_column_names, - context, - false); - } + if ((all_parts_are_remote && settings.allow_prefetched_read_pool_for_remote_filesystem + && MergeTreePrefetchedReadPool::checkReadMethodAllowed(reader_settings.read_settings.remote_fs_method)) + || (all_parts_are_local && settings.allow_prefetched_read_pool_for_local_filesystem + && MergeTreePrefetchedReadPool::checkReadMethodAllowed(reader_settings.read_settings.local_fs_method))) + { + pool = std::make_shared( + max_streams, + sum_marks, + min_marks_for_concurrent_read, + std::move(parts_with_range), + storage_snapshot, + prewhere_info, + actions_settings, + required_columns, + virt_column_names, + settings.preferred_block_size_bytes, + reader_settings, + context, + use_uncompressed_cache, + all_parts_are_remote, + *data.getSettings()); + } + else + { + pool = std::make_shared( + max_streams, + sum_marks, + min_marks_for_concurrent_read, + std::move(parts_with_range), + storage_snapshot, + prewhere_info, + actions_settings, + reader_settings, + required_columns, + virt_column_names, + context, + false); + } auto * logger = &Poco::Logger::get(data.getLogName() + " (SelectExecutor)"); LOG_DEBUG(logger, "Reading approx. {} rows with {} streams", total_rows, max_streams); @@ -1732,6 +1730,36 @@ void ReadFromMergeTree::describeActions(FormatSettings & format_settings) const format_settings.out << prefix << "Parts: " << result.index_stats.back().num_parts_after << '\n'; format_settings.out << prefix << "Granules: " << result.index_stats.back().num_granules_after << '\n'; } + + if (prewhere_info) + { + format_settings.out << prefix << "Prewhere info" << '\n'; + format_settings.out << prefix << "Need filter: " << prewhere_info->need_filter << '\n'; + + prefix.push_back(format_settings.indent_char); + prefix.push_back(format_settings.indent_char); + + if (prewhere_info->prewhere_actions) + { + format_settings.out << prefix << "Prewhere filter" << '\n'; + format_settings.out << prefix << "Prewhere filter column: " << prewhere_info->prewhere_column_name; + if (prewhere_info->remove_prewhere_column) + format_settings.out << " (removed)"; + format_settings.out << '\n'; + + auto expression = std::make_shared(prewhere_info->prewhere_actions); + expression->describeActions(format_settings.out, prefix); + } + + if (prewhere_info->row_level_filter) + { + format_settings.out << prefix << "Row level filter" << '\n'; + format_settings.out << prefix << "Row level filter column: " << prewhere_info->row_level_column_name << '\n'; + + auto expression = std::make_shared(prewhere_info->row_level_filter); + expression->describeActions(format_settings.out, prefix); + } + } } void ReadFromMergeTree::describeActions(JSONBuilder::JSONMap & map) const @@ -1743,6 +1771,35 @@ void ReadFromMergeTree::describeActions(JSONBuilder::JSONMap & map) const map.add("Parts", result.index_stats.back().num_parts_after); map.add("Granules", result.index_stats.back().num_granules_after); } + + if (prewhere_info) + { + std::unique_ptr prewhere_info_map = std::make_unique(); + prewhere_info_map->add("Need filter", prewhere_info->need_filter); + + if (prewhere_info->prewhere_actions) + { + std::unique_ptr prewhere_filter_map = std::make_unique(); + prewhere_filter_map->add("Prewhere filter column", prewhere_info->prewhere_column_name); + prewhere_filter_map->add("Prewhere filter remove filter column", prewhere_info->remove_prewhere_column); + auto expression = std::make_shared(prewhere_info->prewhere_actions); + prewhere_filter_map->add("Prewhere filter expression", expression->toTree()); + + prewhere_info_map->add("Prewhere filter", std::move(prewhere_filter_map)); + } + + if (prewhere_info->row_level_filter) + { + std::unique_ptr row_level_filter_map = std::make_unique(); + row_level_filter_map->add("Row level filter column", prewhere_info->row_level_column_name); + auto expression = std::make_shared(prewhere_info->row_level_filter); + row_level_filter_map->add("Row level filter expression", expression->toTree()); + + prewhere_info_map->add("Row level filter", std::move(row_level_filter_map)); + } + + map.add("Prewhere info", std::move(prewhere_info_map)); + } } void ReadFromMergeTree::describeIndexes(FormatSettings & format_settings) const diff --git a/src/Processors/QueryPlan/ReadFromRemote.cpp b/src/Processors/QueryPlan/ReadFromRemote.cpp index 6a3670f964e..63325488e90 100644 --- a/src/Processors/QueryPlan/ReadFromRemote.cpp +++ b/src/Processors/QueryPlan/ReadFromRemote.cpp @@ -202,7 +202,7 @@ void ReadFromRemote::addLazyPipe(Pipes & pipes, const ClusterProxy::SelectStream scalars["_shard_num"] = Block{{DataTypeUInt32().createColumnConst(1, shard.shard_info.shard_num), std::make_shared(), "_shard_num"}}; auto remote_query_executor = std::make_shared( - shard.shard_info.pool, std::move(connections), query_string, header, context, throttler, scalars, external_tables, stage); + std::move(connections), query_string, header, context, throttler, scalars, external_tables, stage); auto pipe = createRemoteSourcePipe(remote_query_executor, add_agg_info, add_totals, add_extremes, async_read); QueryPipelineBuilder builder; diff --git a/src/Processors/QueryPlan/RollupStep.cpp b/src/Processors/QueryPlan/RollupStep.cpp index 3305f24602f..136690ccfc0 100644 --- a/src/Processors/QueryPlan/RollupStep.cpp +++ b/src/Processors/QueryPlan/RollupStep.cpp @@ -11,7 +11,6 @@ static ITransformingStep::Traits getTraits() return ITransformingStep::Traits { { - .preserves_distinct_columns = false, .returns_single_stream = true, .preserves_number_of_streams = false, .preserves_sorting = false, @@ -29,9 +28,6 @@ RollupStep::RollupStep(const DataStream & input_stream_, Aggregator::Params para , final(final_) , use_nulls(use_nulls_) { - /// Aggregation keys are distinct - for (const auto & key : params.keys) - output_stream->distinct_columns.insert(key); } ProcessorPtr addGroupingSetForTotals(const Block & header, const Names & keys, bool use_nulls, const BuildQueryPipelineSettings & settings, UInt64 grouping_set_number); @@ -54,10 +50,6 @@ void RollupStep::updateOutputStream() { output_stream = createOutputStream( input_streams.front(), appendGroupingSetColumn(params.getHeader(input_streams.front().header, final)), getDataStreamTraits()); - - /// Aggregation keys are distinct - for (const auto & key : params.keys) - output_stream->distinct_columns.insert(key); } diff --git a/src/Processors/QueryPlan/SortingStep.cpp b/src/Processors/QueryPlan/SortingStep.cpp index 0308e320e3a..0ab8e091e05 100644 --- a/src/Processors/QueryPlan/SortingStep.cpp +++ b/src/Processors/QueryPlan/SortingStep.cpp @@ -45,7 +45,6 @@ static ITransformingStep::Traits getTraits(size_t limit) return ITransformingStep::Traits { { - .preserves_distinct_columns = true, .returns_single_stream = true, .preserves_number_of_streams = false, .preserves_sorting = false, diff --git a/src/Processors/QueryPlan/TotalsHavingStep.cpp b/src/Processors/QueryPlan/TotalsHavingStep.cpp index 63991655426..d1bd70fd0b2 100644 --- a/src/Processors/QueryPlan/TotalsHavingStep.cpp +++ b/src/Processors/QueryPlan/TotalsHavingStep.cpp @@ -14,7 +14,6 @@ static ITransformingStep::Traits getTraits(bool has_filter) return ITransformingStep::Traits { { - .preserves_distinct_columns = true, .returns_single_stream = true, .preserves_number_of_streams = false, .preserves_sorting = true, diff --git a/src/Processors/QueryPlan/WindowStep.cpp b/src/Processors/QueryPlan/WindowStep.cpp index 92e9948c4c7..d313b210854 100644 --- a/src/Processors/QueryPlan/WindowStep.cpp +++ b/src/Processors/QueryPlan/WindowStep.cpp @@ -15,7 +15,6 @@ static ITransformingStep::Traits getTraits() return ITransformingStep::Traits { { - .preserves_distinct_columns = true, .returns_single_stream = false, .preserves_number_of_streams = true, .preserves_sorting = true, diff --git a/src/Processors/Transforms/FillingTransform.cpp b/src/Processors/Transforms/FillingTransform.cpp index e0c79d50141..4a729863200 100644 --- a/src/Processors/Transforms/FillingTransform.cpp +++ b/src/Processors/Transforms/FillingTransform.cpp @@ -169,17 +169,13 @@ static bool tryConvertFields(FillColumnDescription & descr, const DataTypePtr & } FillingTransform::FillingTransform( - const Block & header_, const SortDescription & sort_description_, InterpolateDescriptionPtr interpolate_description_, bool on_totals_) + const Block & header_, const SortDescription & sort_description_, InterpolateDescriptionPtr interpolate_description_) : ISimpleTransform(header_, transformHeader(header_, sort_description_), true) , sort_description(sort_description_) , interpolate_description(interpolate_description_) - , on_totals(on_totals_) , filling_row(sort_description_) , next_row(sort_description_) { - if (on_totals) - return; - if (interpolate_description) interpolate_actions = std::make_shared(interpolate_description->actions); @@ -239,7 +235,7 @@ FillingTransform::FillingTransform( IProcessor::Status FillingTransform::prepare() { - if (!on_totals && input.isFinished() && !output.isFinished() && !has_input && !generate_suffix) + if (input.isFinished() && !output.isFinished() && !has_input && !generate_suffix) { should_insert_first = next_row < filling_row || first; @@ -263,109 +259,175 @@ IProcessor::Status FillingTransform::prepare() return ISimpleTransform::prepare(); } +void FillingTransform::interpolate(const MutableColumns & result_columns, Block & interpolate_block) +{ + if (interpolate_description) + { + interpolate_block.clear(); + + if (!input_positions.empty()) + { + /// populate calculation block with required columns with values from previous row + for (const auto & [col_pos, name_type] : input_positions) + { + MutableColumnPtr column = name_type.type->createColumn(); + const auto * res_column = result_columns[col_pos].get(); + size_t size = res_column->size(); + if (size == 0) /// this is the first row in current chunk + { + /// take value from last row of previous chunk if exists, else use default + if (last_row.size() > col_pos && !last_row[col_pos]->empty()) + column->insertFrom(*last_row[col_pos], 0); + else + column->insertDefault(); + } + else /// take value from previous row of current chunk + column->insertFrom(*res_column, size - 1); + + interpolate_block.insert({std::move(column), name_type.type, name_type.name}); + } + interpolate_actions->execute(interpolate_block); + } + else /// all INTERPOLATE expressions are constants + { + size_t n = 1; + interpolate_actions->execute(interpolate_block, n); + } + } +} + +using MutableColumnRawPtrs = std::vector; + +static void insertFromFillingRow(const MutableColumnRawPtrs & filling_columns, const MutableColumnRawPtrs & interpolate_columns, const MutableColumnRawPtrs & other_columns, + const FillingRow & filling_row, const Block & interpolate_block) +{ + for (size_t i = 0, size = filling_columns.size(); i < size; ++i) + { + if (filling_row[i].isNull()) + filling_columns[i]->insertDefault(); + else + filling_columns[i]->insert(filling_row[i]); + } + + if (size_t size = interpolate_block.columns()) + { + Columns columns = interpolate_block.getColumns(); + for (size_t i = 0; i < size; ++i) + interpolate_columns[i]->insertFrom(*columns[i]->convertToFullColumnIfConst(), 0); + } + else + for (auto * interpolate_column : interpolate_columns) + interpolate_column->insertDefault(); + + for (auto * other_column : other_columns) + other_column->insertDefault(); +} + +static void copyRowFromColumns(const MutableColumnRawPtrs & dest, const Columns & source, size_t row_num) +{ + for (size_t i = 0, size = source.size(); i < size; ++i) + dest[i]->insertFrom(*source[i], row_num); +} + +static void initColumnsByPositions( + const Columns & input_columns, + Columns & input_columns_by_positions, + const MutableColumns & output_columns, + MutableColumnRawPtrs & output_columns_by_position, + const std::vector & positions) +{ + for (size_t pos : positions) + { + input_columns_by_positions.push_back(input_columns[pos]); + output_columns_by_position.push_back(output_columns[pos].get()); + } +} + +void FillingTransform::initColumns( + const Columns & input_columns, + Columns & input_fill_columns, + Columns & input_interpolate_columns, + Columns & input_other_columns, + MutableColumns & output_columns, + MutableColumnRawPtrs & output_fill_columns, + MutableColumnRawPtrs & output_interpolate_columns, + MutableColumnRawPtrs & output_other_columns) +{ + Columns non_const_columns; + non_const_columns.reserve(input_columns.size()); + + for (const auto & column : input_columns) + non_const_columns.push_back(column->convertToFullColumnIfConst()); + + for (const auto & column : non_const_columns) + output_columns.push_back(column->cloneEmpty()->assumeMutable()); + + initColumnsByPositions(non_const_columns, input_fill_columns, output_columns, output_fill_columns, fill_column_positions); + initColumnsByPositions( + non_const_columns, input_interpolate_columns, output_columns, output_interpolate_columns, interpolate_column_positions); + initColumnsByPositions(non_const_columns, input_other_columns, output_columns, output_other_columns, other_column_positions); +} void FillingTransform::transform(Chunk & chunk) { - if (on_totals) - return; - if (!chunk.hasRows() && !generate_suffix) return; Columns old_fill_columns; Columns old_interpolate_columns; Columns old_other_columns; - MutableColumns res_fill_columns; - MutableColumns res_interpolate_columns; - MutableColumns res_other_columns; - - std::vector> res_map; - res_map.resize(input.getHeader().columns()); - - auto init_columns_by_positions = [&res_map](const Columns & old_columns, Columns & new_columns, - MutableColumns & new_mutable_columns, const Positions & positions) - { - for (size_t pos : positions) - { - auto old_column = old_columns[pos]->convertToFullColumnIfConst(); - new_columns.push_back(old_column); - res_map[pos] = {&new_mutable_columns, new_mutable_columns.size()}; - new_mutable_columns.push_back(old_column->cloneEmpty()->assumeMutable()); - } - }; + MutableColumnRawPtrs res_fill_columns; + MutableColumnRawPtrs res_interpolate_columns; + MutableColumnRawPtrs res_other_columns; + MutableColumns result_columns; Block interpolate_block; - auto interpolate = [&]() - { - if (interpolate_description) - { - interpolate_block.clear(); - - if (!input_positions.empty()) - { - /// populate calculation block with required columns with values from previous row - for (const auto & [col_pos, name_type] : input_positions) - { - MutableColumnPtr column = name_type.type->createColumn(); - auto [res_columns, pos] = res_map[col_pos]; - size_t size = (*res_columns)[pos]->size(); - if (size == 0) /// this is the first row in current chunk - { - /// take value from last row of previous chunk if exists, else use default - if (last_row.size() > col_pos && !last_row[col_pos]->empty()) - column->insertFrom(*last_row[col_pos], 0); - else - column->insertDefault(); - } - else /// take value from previous row of current chunk - column->insertFrom(*(*res_columns)[pos], size - 1); - - interpolate_block.insert({std::move(column), name_type.type, name_type.name}); - } - interpolate_actions->execute(interpolate_block); - } - else /// all INTERPOLATE expressions are constants - { - size_t n = 1; - interpolate_actions->execute(interpolate_block, n); - } - } - }; - if (generate_suffix) { const auto & empty_columns = input.getHeader().getColumns(); - init_columns_by_positions(empty_columns, old_fill_columns, res_fill_columns, fill_column_positions); - init_columns_by_positions(empty_columns, old_interpolate_columns, res_interpolate_columns, interpolate_column_positions); - init_columns_by_positions(empty_columns, old_other_columns, res_other_columns, other_column_positions); + initColumns( + empty_columns, + old_fill_columns, + old_interpolate_columns, + old_other_columns, + result_columns, + res_fill_columns, + res_interpolate_columns, + res_other_columns); if (first) filling_row.initFromDefaults(); if (should_insert_first && filling_row < next_row) { - interpolate(); + interpolate(result_columns, interpolate_block); insertFromFillingRow(res_fill_columns, res_interpolate_columns, res_other_columns, filling_row, interpolate_block); } - interpolate(); + interpolate(result_columns, interpolate_block); while (filling_row.next(next_row)) { insertFromFillingRow(res_fill_columns, res_interpolate_columns, res_other_columns, filling_row, interpolate_block); - interpolate(); + interpolate(result_columns, interpolate_block); } - setResultColumns(chunk, res_fill_columns, res_interpolate_columns, res_other_columns); + size_t num_output_rows = result_columns[0]->size(); + chunk.setColumns(std::move(result_columns), num_output_rows); return; } - size_t num_rows = chunk.getNumRows(); + const size_t num_rows = chunk.getNumRows(); auto old_columns = chunk.detachColumns(); - - init_columns_by_positions(old_columns, old_fill_columns, res_fill_columns, fill_column_positions); - init_columns_by_positions(old_columns, old_interpolate_columns, res_interpolate_columns, interpolate_column_positions); - init_columns_by_positions(old_columns, old_other_columns, res_other_columns, other_column_positions); + initColumns( + old_columns, + old_fill_columns, + old_interpolate_columns, + old_other_columns, + result_columns, + res_fill_columns, + res_interpolate_columns, + res_other_columns); if (first) { @@ -379,7 +441,7 @@ void FillingTransform::transform(Chunk & chunk) filling_row.initFromDefaults(i); if (less(fill_from, current_value, filling_row.getDirection(i))) { - interpolate(); + interpolate(result_columns, interpolate_block); insertFromFillingRow(res_fill_columns, res_interpolate_columns, res_other_columns, filling_row, interpolate_block); } break; @@ -393,7 +455,7 @@ void FillingTransform::transform(Chunk & chunk) { should_insert_first = next_row < filling_row; - for (size_t i = 0; i < filling_row.size(); ++i) + for (size_t i = 0, size = filling_row.size(); i < size; ++i) { auto current_value = (*old_fill_columns[i])[row_ind]; const auto & fill_to = filling_row.getFillDescription(i).fill_to; @@ -408,15 +470,15 @@ void FillingTransform::transform(Chunk & chunk) /// and probably we need to insert it to block. if (should_insert_first && filling_row < next_row) { - interpolate(); + interpolate(result_columns, interpolate_block); insertFromFillingRow(res_fill_columns, res_interpolate_columns, res_other_columns, filling_row, interpolate_block); } - interpolate(); + interpolate(result_columns, interpolate_block); while (filling_row.next(next_row)) { insertFromFillingRow(res_fill_columns, res_interpolate_columns, res_other_columns, filling_row, interpolate_block); - interpolate(); + interpolate(result_columns, interpolate_block); } copyRowFromColumns(res_fill_columns, old_fill_columns, row_ind); @@ -424,55 +486,24 @@ void FillingTransform::transform(Chunk & chunk) copyRowFromColumns(res_other_columns, old_other_columns, row_ind); } - saveLastRow(res_fill_columns, res_interpolate_columns, res_other_columns); - setResultColumns(chunk, res_fill_columns, res_interpolate_columns, res_other_columns); + saveLastRow(result_columns); + size_t num_output_rows = result_columns[0]->size(); + chunk.setColumns(std::move(result_columns), num_output_rows); } -void FillingTransform::setResultColumns(Chunk & chunk, MutableColumns & fill_columns, MutableColumns & interpolate_columns, MutableColumns & other_columns) const -{ - MutableColumns result_columns(fill_columns.size() + interpolate_columns.size() + other_columns.size()); - /// fill_columns always non-empty. - size_t num_rows = fill_columns[0]->size(); - - for (size_t i = 0, size = fill_columns.size(); i < size; ++i) - result_columns[fill_column_positions[i]] = std::move(fill_columns[i]); - for (size_t i = 0, size = interpolate_columns.size(); i < size; ++i) - result_columns[interpolate_column_positions[i]] = std::move(interpolate_columns[i]); - for (size_t i = 0, size = other_columns.size(); i < size; ++i) - result_columns[other_column_positions[i]] = std::move(other_columns[i]); - - chunk.setColumns(std::move(result_columns), num_rows); -} - -void FillingTransform::saveLastRow(const MutableColumns & fill_columns, const MutableColumns & interpolate_columns, const MutableColumns & other_columns) +void FillingTransform::saveLastRow(const MutableColumns & result_columns) { last_row.clear(); - last_row.resize(fill_columns.size() + interpolate_columns.size() + other_columns.size()); - size_t num_rows = fill_columns[0]->size(); + const size_t num_rows = result_columns[0]->size(); if (num_rows == 0) return; - for (size_t i = 0, size = fill_columns.size(); i < size; ++i) + for (const auto & result_column : result_columns) { - auto column = fill_columns[i]->cloneEmpty(); - column->insertFrom(*fill_columns[i], num_rows - 1); - last_row[fill_column_positions[i]] = std::move(column); - } - - for (size_t i = 0, size = interpolate_columns.size(); i < size; ++i) - { - auto column = interpolate_columns[i]->cloneEmpty(); - column->insertFrom(*interpolate_columns[i], num_rows - 1); - last_row[interpolate_column_positions[i]] = std::move(column); - } - - for (size_t i = 0, size = other_columns.size(); i < size; ++i) - { - auto column = other_columns[i]->cloneEmpty(); - column->insertFrom(*other_columns[i], num_rows - 1); - last_row[other_column_positions[i]] = std::move(column); + auto column = result_column->cloneEmpty(); + column->insertFrom(*result_column, num_rows - 1); + last_row.push_back(std::move(column)); } } - } diff --git a/src/Processors/Transforms/FillingTransform.h b/src/Processors/Transforms/FillingTransform.h index 7b41ab795d1..5331254b08c 100644 --- a/src/Processors/Transforms/FillingTransform.h +++ b/src/Processors/Transforms/FillingTransform.h @@ -16,7 +16,7 @@ namespace DB class FillingTransform : public ISimpleTransform { public: - FillingTransform(const Block & header_, const SortDescription & sort_description_, InterpolateDescriptionPtr interpolate_description_, bool on_totals_); + FillingTransform(const Block & header_, const SortDescription & sort_description_, InterpolateDescriptionPtr interpolate_description_); String getName() const override { return "FillingTransform"; } @@ -28,12 +28,22 @@ protected: void transform(Chunk & Chunk) override; private: - void setResultColumns(Chunk & chunk, MutableColumns & fill_columns, MutableColumns & interpolate_columns, MutableColumns & other_columns) const; - void saveLastRow(const MutableColumns & fill_columns, const MutableColumns & interpolate_columns, const MutableColumns & other_columns); + void saveLastRow(const MutableColumns & result_columns); + void interpolate(const MutableColumns& result_columns, Block & interpolate_block); + + using MutableColumnRawPtrs = std::vector; + void initColumns( + const Columns & input_columns, + Columns & input_fill_columns, + Columns & input_interpolate_columns, + Columns & input_other_columns, + MutableColumns & output_columns, + MutableColumnRawPtrs & output_fill_columns, + MutableColumnRawPtrs & output_interpolate_columns, + MutableColumnRawPtrs & output_other_columns); const SortDescription sort_description; /// Contains only columns with WITH FILL. const InterpolateDescriptionPtr interpolate_description; /// Contains INTERPOLATE columns - const bool on_totals; /// FillingTransform does nothing on totals. FillingRow filling_row; /// Current row, which is used to fill gaps. FillingRow next_row; /// Row to which we need to generate filling rows. @@ -53,4 +63,16 @@ private: bool should_insert_first = false; }; +class FillingNoopTransform : public ISimpleTransform +{ +public: + FillingNoopTransform(const Block & header, const SortDescription & sort_description_) + : ISimpleTransform(header, FillingTransform::transformHeader(header, sort_description_), true) + { + } + + void transform(Chunk &) override {} + String getName() const override { return "FillingNoopTransform"; } +}; + } diff --git a/src/Processors/Transforms/JoiningTransform.cpp b/src/Processors/Transforms/JoiningTransform.cpp index c28a84e9d5d..bba8ec6fa16 100644 --- a/src/Processors/Transforms/JoiningTransform.cpp +++ b/src/Processors/Transforms/JoiningTransform.cpp @@ -318,13 +318,24 @@ DelayedJoinedBlocksWorkerTransform::DelayedJoinedBlocksWorkerTransform(Block out IProcessor::Status DelayedJoinedBlocksWorkerTransform::prepare() { + auto & output = outputs.front(); + auto & input = inputs.front(); + + if (output.isFinished()) + { + input.close(); + return Status::Finished; + } + + if (!output.canPush()) + { + input.setNotNeeded(); + return Status::PortFull; + } + if (inputs.size() != 1 && outputs.size() != 1) throw Exception(ErrorCodes::LOGICAL_ERROR, "DelayedJoinedBlocksWorkerTransform must have exactly one input port"); - auto & output = outputs.front(); - - auto & input = inputs.front(); - if (output_chunk) { input.setNotNeeded(); @@ -397,15 +408,25 @@ DelayedJoinedBlocksTransform::DelayedJoinedBlocksTransform(size_t num_streams, J void DelayedJoinedBlocksTransform::work() { + if (finished) + return; + delayed_blocks = join->getDelayedBlocks(); finished = finished || delayed_blocks == nullptr; } - IProcessor::Status DelayedJoinedBlocksTransform::prepare() { for (auto & output : outputs) { + if (output.isFinished()) + { + /// If at least one output is finished, then we have read all data from buckets. + /// Some workers can still be busy with joining the last chunk of data in memory, + /// but after that they also will finish when they will try to get next chunk. + finished = true; + continue; + } if (!output.canPush()) return Status::PortFull; } @@ -414,6 +435,8 @@ IProcessor::Status DelayedJoinedBlocksTransform::prepare() { for (auto & output : outputs) { + if (output.isFinished()) + continue; Chunk chunk; chunk.setChunkInfo(std::make_shared()); output.push(std::move(chunk)); diff --git a/src/Processors/Transforms/buildPushingToViewsChain.cpp b/src/Processors/Transforms/buildPushingToViewsChain.cpp index 870106d794f..c27e73804ad 100644 --- a/src/Processors/Transforms/buildPushingToViewsChain.cpp +++ b/src/Processors/Transforms/buildPushingToViewsChain.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -710,6 +711,7 @@ IProcessor::Status FinalizingViewsTransform::prepare() if (!output.canPush()) return Status::PortFull; + bool materialized_views_ignore_errors = views_data->context->getSettingsRef().materialized_views_ignore_errors; size_t num_finished = 0; size_t pos = 0; for (auto & input : inputs) @@ -735,7 +737,7 @@ IProcessor::Status FinalizingViewsTransform::prepare() else statuses[i].exception = data.exception; - if (i == 0 && statuses[0].is_first) + if (i == 0 && statuses[0].is_first && !materialized_views_ignore_errors) { output.pushData(std::move(data)); return Status::PortFull; @@ -752,7 +754,7 @@ IProcessor::Status FinalizingViewsTransform::prepare() if (!statuses.empty()) return Status::Ready; - if (any_exception) + if (any_exception && !materialized_views_ignore_errors) output.pushException(any_exception); output.finish(); @@ -782,6 +784,8 @@ static std::exception_ptr addStorageToException(std::exception_ptr ptr, const St void FinalizingViewsTransform::work() { + bool materialized_views_ignore_errors = views_data->context->getSettingsRef().materialized_views_ignore_errors; + size_t i = 0; for (auto & view : views_data->views) { @@ -794,6 +798,10 @@ void FinalizingViewsTransform::work() any_exception = status.exception; view.setException(addStorageToException(status.exception, view.table_id)); + + /// Exception will be ignored, it is saved here for the system.query_views_log + if (materialized_views_ignore_errors) + tryLogException(view.exception, &Poco::Logger::get("PushingToViews"), "Cannot push to the storage, ignoring the error"); } else { diff --git a/src/QueryPipeline/BlockIO.cpp b/src/QueryPipeline/BlockIO.cpp index 9af7cd2b772..231c369707e 100644 --- a/src/QueryPipeline/BlockIO.cpp +++ b/src/QueryPipeline/BlockIO.cpp @@ -4,6 +4,10 @@ namespace DB { +namespace ErrorCodes +{ + extern const int QUERY_WAS_CANCELLED; +} void BlockIO::reset() { @@ -58,7 +62,26 @@ void BlockIO::onFinish() void BlockIO::onException() { if (exception_callback) - exception_callback(); + exception_callback(/* log_error */ true); + + pipeline.reset(); +} + +void BlockIO::onCancelOrConnectionLoss() +{ + /// Query was not finished gracefully, so we should call exception_callback + /// But we don't have a real exception + if (exception_callback) + { + try + { + throw Exception(ErrorCodes::QUERY_WAS_CANCELLED, "Query was cancelled or a client has unexpectedly dropped the connection"); + } + catch (...) + { + exception_callback(/* log_error */ false); + } + } pipeline.reset(); } diff --git a/src/QueryPipeline/BlockIO.h b/src/QueryPipeline/BlockIO.h index 4c8d29d0ba8..ff85a0d6772 100644 --- a/src/QueryPipeline/BlockIO.h +++ b/src/QueryPipeline/BlockIO.h @@ -26,13 +26,14 @@ struct BlockIO /// Callbacks for query logging could be set here. std::function finish_callback; - std::function exception_callback; + std::function exception_callback; /// When it is true, don't bother sending any non-empty blocks to the out stream bool null_format = false; void onFinish(); void onException(); + void onCancelOrConnectionLoss(); /// Set is_all_data_sent in system.processes for this query. void setAllDataSent() const; diff --git a/src/QueryPipeline/ConnectionCollector.cpp b/src/QueryPipeline/ConnectionCollector.cpp deleted file mode 100644 index 7c484dcd6e8..00000000000 --- a/src/QueryPipeline/ConnectionCollector.cpp +++ /dev/null @@ -1,124 +0,0 @@ -#include - -#include -#include -#include -#include -#include "Core/Protocol.h" -#include - -namespace CurrentMetrics -{ -extern const Metric AsyncDrainedConnections; -extern const Metric ActiveAsyncDrainedConnections; -} - -namespace DB -{ -namespace ErrorCodes -{ - extern const int LOGICAL_ERROR; - extern const int UNEXPECTED_PACKET_FROM_SERVER; -} - -std::unique_ptr ConnectionCollector::connection_collector; - -static constexpr UInt64 max_connection_draining_tasks_per_thread = 20; - -ConnectionCollector::ConnectionCollector(ContextMutablePtr global_context_, size_t max_threads) - : WithMutableContext(global_context_), pool(max_threads, max_threads, max_threads * max_connection_draining_tasks_per_thread) -{ -} - -ConnectionCollector & ConnectionCollector::init(ContextMutablePtr global_context_, size_t max_threads) -{ - if (connection_collector) - { - throw Exception(ErrorCodes::LOGICAL_ERROR, "Connection collector is initialized twice. This is a bug"); - } - - connection_collector.reset(new ConnectionCollector(global_context_, max_threads)); - return *connection_collector; -} - -struct AsyncDrainTask -{ - const ConnectionPoolWithFailoverPtr pool; - std::shared_ptr shared_connections; - void operator()() const - { - ConnectionCollector::drainConnections(*shared_connections, /* throw_error= */ false); - } - - // We don't have std::unique_function yet. Wrap it in shared_ptr to make the functor copyable. - std::shared_ptr metric_increment - = std::make_shared(CurrentMetrics::ActiveAsyncDrainedConnections); -}; - -std::shared_ptr ConnectionCollector::enqueueConnectionCleanup( - const ConnectionPoolWithFailoverPtr & pool, std::shared_ptr connections) noexcept -{ - if (!connections) - return nullptr; - - if (connection_collector) - { - if (connection_collector->pool.trySchedule(AsyncDrainTask{pool, connections})) - { - CurrentMetrics::add(CurrentMetrics::AsyncDrainedConnections, 1); - return nullptr; - } - } - return connections; -} - -void ConnectionCollector::drainConnections(IConnections & connections, bool throw_error) -{ - bool is_drained = false; - try - { - Packet packet = connections.drain(); - is_drained = true; - switch (packet.type) - { - case Protocol::Server::EndOfStream: - case Protocol::Server::Log: - case Protocol::Server::ProfileEvents: - break; - - case Protocol::Server::Exception: - packet.exception->rethrow(); - break; - - default: - /// Connection should be closed in case of unexpected packet, - /// since this means that the connection in some bad state. - is_drained = false; - throw NetException( - ErrorCodes::UNEXPECTED_PACKET_FROM_SERVER, - "Unexpected packet {} from one of the following replicas: {}. (expected EndOfStream, Log, ProfileEvents or Exception)", - Protocol::Server::toString(packet.type), - connections.dumpAddresses()); - } - } - catch (...) - { - tryLogCurrentException(&Poco::Logger::get("ConnectionCollector"), __PRETTY_FUNCTION__); - if (!is_drained) - { - try - { - connections.disconnect(); - } - catch (...) - { - tryLogCurrentException(&Poco::Logger::get("ConnectionCollector"), __PRETTY_FUNCTION__); - } - } - - if (throw_error) - throw; - } -} - -} diff --git a/src/QueryPipeline/ConnectionCollector.h b/src/QueryPipeline/ConnectionCollector.h deleted file mode 100644 index 44482607277..00000000000 --- a/src/QueryPipeline/ConnectionCollector.h +++ /dev/null @@ -1,30 +0,0 @@ -#pragma once - -#include -#include -#include -#include - -namespace DB -{ - -class ConnectionPoolWithFailover; -using ConnectionPoolWithFailoverPtr = std::shared_ptr; - -class ConnectionCollector : boost::noncopyable, WithMutableContext -{ -public: - static ConnectionCollector & init(ContextMutablePtr global_context_, size_t max_threads); - static std::shared_ptr - enqueueConnectionCleanup(const ConnectionPoolWithFailoverPtr & pool, std::shared_ptr connections) noexcept; - static void drainConnections(IConnections & connections, bool throw_error); - -private: - explicit ConnectionCollector(ContextMutablePtr global_context_, size_t max_threads); - - static constexpr size_t reschedule_time_ms = 1000; - ThreadPool pool; - static std::unique_ptr connection_collector; -}; - -} diff --git a/src/QueryPipeline/RemoteQueryExecutor.cpp b/src/QueryPipeline/RemoteQueryExecutor.cpp index a10d70d22e9..e3a69958213 100644 --- a/src/QueryPipeline/RemoteQueryExecutor.cpp +++ b/src/QueryPipeline/RemoteQueryExecutor.cpp @@ -1,6 +1,4 @@ #include - -#include #include #include @@ -23,14 +21,9 @@ #include #include #include +#include -namespace CurrentMetrics -{ - extern const Metric SyncDrainedConnections; - extern const Metric ActiveSyncDrainedConnections; -} - namespace ProfileEvents { extern const Event ReadTaskRequestsReceived; @@ -67,7 +60,7 @@ RemoteQueryExecutor::RemoteQueryExecutor( { create_connections = [this, &connection, throttler, extension_]() { - auto res = std::make_shared(connection, context->getSettingsRef(), throttler); + auto res = std::make_unique(connection, context->getSettingsRef(), throttler); if (extension_ && extension_->replica_info) res->setReplicaInfo(*extension_->replica_info); return res; @@ -83,7 +76,7 @@ RemoteQueryExecutor::RemoteQueryExecutor( { create_connections = [this, connection_ptr, throttler, extension_]() { - auto res = std::make_shared(connection_ptr, context->getSettingsRef(), throttler); + auto res = std::make_unique(connection_ptr, context->getSettingsRef(), throttler); if (extension_ && extension_->replica_info) res->setReplicaInfo(*extension_->replica_info); return res; @@ -91,7 +84,6 @@ RemoteQueryExecutor::RemoteQueryExecutor( } RemoteQueryExecutor::RemoteQueryExecutor( - const ConnectionPoolWithFailoverPtr & pool_, std::vector && connections_, const String & query_, const Block & header_, ContextPtr context_, const ThrottlerPtr & throttler, const Scalars & scalars_, const Tables & external_tables_, @@ -100,10 +92,9 @@ RemoteQueryExecutor::RemoteQueryExecutor( , scalars(scalars_), external_tables(external_tables_), stage(stage_) , task_iterator(extension_ ? extension_->task_iterator : nullptr) , parallel_reading_coordinator(extension_ ? extension_->parallel_reading_coordinator : nullptr) - , pool(pool_) { create_connections = [this, connections_, throttler, extension_]() mutable { - auto res = std::make_shared(std::move(connections_), context->getSettingsRef(), throttler); + auto res = std::make_unique(std::move(connections_), context->getSettingsRef(), throttler); if (extension_ && extension_->replica_info) res->setReplicaInfo(*extension_->replica_info); return res; @@ -111,7 +102,7 @@ RemoteQueryExecutor::RemoteQueryExecutor( } RemoteQueryExecutor::RemoteQueryExecutor( - const ConnectionPoolWithFailoverPtr & pool_, + const ConnectionPoolWithFailoverPtr & pool, const String & query_, const Block & header_, ContextPtr context_, const ThrottlerPtr & throttler, const Scalars & scalars_, const Tables & external_tables_, QueryProcessingStage::Enum stage_, std::optional extension_) @@ -119,9 +110,8 @@ RemoteQueryExecutor::RemoteQueryExecutor( , scalars(scalars_), external_tables(external_tables_), stage(stage_) , task_iterator(extension_ ? extension_->task_iterator : nullptr) , parallel_reading_coordinator(extension_ ? extension_->parallel_reading_coordinator : nullptr) - , pool(pool_) { - create_connections = [this, throttler, extension_]()->std::shared_ptr + create_connections = [this, pool, throttler, extension_]()->std::unique_ptr { const Settings & current_settings = context->getSettingsRef(); auto timeouts = ConnectionTimeouts::getTCPTimeoutsWithFailover(current_settings); @@ -133,7 +123,7 @@ RemoteQueryExecutor::RemoteQueryExecutor( if (main_table) table_to_check = std::make_shared(main_table.getQualifiedName()); - auto res = std::make_shared(pool, context, timeouts, throttler, pool_mode, table_to_check); + auto res = std::make_unique(pool, context, timeouts, throttler, pool_mode, table_to_check); if (extension_ && extension_->replica_info) res->setReplicaInfo(*extension_->replica_info); return res; @@ -151,7 +141,7 @@ RemoteQueryExecutor::RemoteQueryExecutor( else connection_entries = pool->getMany(timeouts, ¤t_settings, pool_mode); - auto res = std::make_shared(std::move(connection_entries), current_settings, throttler); + auto res = std::make_unique(std::move(connection_entries), current_settings, throttler); if (extension_ && extension_->replica_info) res->setReplicaInfo(*extension_->replica_info); return res; @@ -535,26 +525,38 @@ void RemoteQueryExecutor::finish(std::unique_ptr * read_context) /// Send the request to abort the execution of the request, if not already sent. tryCancel("Cancelling query because enough data has been read", read_context); - if (context->getSettingsRef().drain_timeout != Poco::Timespan(-1000000)) + /// Get the remaining packets so that there is no out of sync in the connections to the replicas. + Packet packet = connections->drain(); + switch (packet.type) { - auto connections_left = ConnectionCollector::enqueueConnectionCleanup(pool, connections); - if (connections_left) - { - /// Drain connections synchronously and suppress errors. - CurrentMetrics::Increment metric_increment(CurrentMetrics::ActiveSyncDrainedConnections); - ConnectionCollector::drainConnections(*connections_left, /* throw_error= */ false); - CurrentMetrics::add(CurrentMetrics::SyncDrainedConnections, 1); - } - } - else - { - /// Drain connections synchronously without suppressing errors. - CurrentMetrics::Increment metric_increment(CurrentMetrics::ActiveSyncDrainedConnections); - ConnectionCollector::drainConnections(*connections, /* throw_error= */ true); - CurrentMetrics::add(CurrentMetrics::SyncDrainedConnections, 1); - } + case Protocol::Server::EndOfStream: + finished = true; + break; - finished = true; + case Protocol::Server::Log: + /// Pass logs from remote server to client + if (auto log_queue = CurrentThread::getInternalTextLogsQueue()) + log_queue->pushBlock(std::move(packet.block)); + break; + + case Protocol::Server::Exception: + got_exception_from_replica = true; + packet.exception->rethrow(); + break; + + case Protocol::Server::ProfileEvents: + /// Pass profile events from remote server to client + if (auto profile_queue = CurrentThread::getInternalProfileEventsQueue()) + if (!profile_queue->emplace(std::move(packet.block))) + throw Exception(ErrorCodes::SYSTEM_ERROR, "Could not push into profile queue"); + break; + + default: + got_unknown_packet_from_replica = true; + throw Exception(ErrorCodes::UNKNOWN_PACKET_FROM_SERVER, "Unknown packet {} from one of the following replicas: {}", + toString(packet.type), + connections->dumpAddresses()); + } } void RemoteQueryExecutor::cancel(std::unique_ptr * read_context) @@ -601,6 +603,9 @@ void RemoteQueryExecutor::sendExternalTables() for (const auto & table : external_tables) { StoragePtr cur = table.second; + /// Send only temporary tables with StorageMemory + if (!std::dynamic_pointer_cast(cur)) + continue; auto data = std::make_unique(); data->table_name = table.first; diff --git a/src/QueryPipeline/RemoteQueryExecutor.h b/src/QueryPipeline/RemoteQueryExecutor.h index c67a45c7275..576efb97bb1 100644 --- a/src/QueryPipeline/RemoteQueryExecutor.h +++ b/src/QueryPipeline/RemoteQueryExecutor.h @@ -52,7 +52,6 @@ public: }; /// Takes already set connection. - /// We don't own connection, thus we have to drain it synchronously. RemoteQueryExecutor( Connection & connection, const String & query_, const Block & header_, ContextPtr context_, @@ -68,7 +67,6 @@ public: /// Accepts several connections already taken from pool. RemoteQueryExecutor( - const ConnectionPoolWithFailoverPtr & pool, std::vector && connections_, const String & query_, const Block & header_, ContextPtr context_, const ThrottlerPtr & throttler = nullptr, const Scalars & scalars_ = Scalars(), const Tables & external_tables_ = Tables(), @@ -191,6 +189,9 @@ private: Block totals; Block extremes; + std::function()> create_connections; + std::unique_ptr connections; + const String query; String query_id; ContextPtr context; @@ -213,12 +214,6 @@ private: /// about the number of the current replica or the count of replicas at all. IConnections::ReplicaInfo replica_info; - std::function()> create_connections; - /// Hold a shared reference to the connection pool so that asynchronous connection draining will - /// work safely. Make sure it's the first member so that we don't destruct it too early. - const ConnectionPoolWithFailoverPtr pool; - std::shared_ptr connections; - /// Streams for reading from temporary tables and following sending of data /// to remote servers for GLOBAL-subqueries std::vector external_tables_data; diff --git a/src/QueryPipeline/RemoteQueryExecutorReadContext.cpp b/src/QueryPipeline/RemoteQueryExecutorReadContext.cpp index 0abf5a26794..0c84e844cb2 100644 --- a/src/QueryPipeline/RemoteQueryExecutorReadContext.cpp +++ b/src/QueryPipeline/RemoteQueryExecutorReadContext.cpp @@ -44,7 +44,7 @@ struct RemoteQueryExecutorRoutine { while (true) { - read_context.packet = connections.receivePacketUnlocked(ReadCallback{read_context, sink}, false /* is_draining */); + read_context.packet = connections.receivePacketUnlocked(ReadCallback{read_context, sink}); sink = std::move(sink).resume(); } } @@ -147,7 +147,7 @@ bool RemoteQueryExecutorReadContext::checkTimeoutImpl(bool blocking) if (is_timer_alarmed && !is_socket_ready) { - /// Socket receive timeout. Drain it in case of error, or it may be hide by timeout exception. + /// Socket receive timeout. Drain it in case or error, or it may be hide by timeout exception. timer.drain(); throw NetException(ErrorCodes::SOCKET_TIMEOUT, "Timeout exceeded"); } diff --git a/src/QueryPipeline/printPipeline.h b/src/QueryPipeline/printPipeline.h index 76143211875..e91909cb50b 100644 --- a/src/QueryPipeline/printPipeline.h +++ b/src/QueryPipeline/printPipeline.h @@ -10,7 +10,6 @@ namespace DB * You can render it with: * dot -T png < pipeline.dot > pipeline.png */ - template void printPipeline(const Processors & processors, const Statuses & statuses, WriteBuffer & out) { @@ -70,5 +69,4 @@ void printPipeline(const Processors & processors, WriteBuffer & out) /// If QueryPlanStep wasn't set for processor, representation may be not correct. /// If with_header is set, prints block header for each edge. void printPipelineCompact(const Processors & processors, WriteBuffer & out, bool with_header); - } diff --git a/src/Server/GRPCServer.cpp b/src/Server/GRPCServer.cpp index 595f5a8c2b7..a6924310c3c 100644 --- a/src/Server/GRPCServer.cpp +++ b/src/Server/GRPCServer.cpp @@ -836,13 +836,14 @@ namespace query_context->applySettingsChanges(settings_changes); query_context->setCurrentQueryId(query_info.query_id()); - query_scope.emplace(query_context); + query_scope.emplace(query_context, /* fatal_error_callback */ [this]{ onFatalError(); }); /// Set up tracing context for this query on current thread thread_trace_context = std::make_unique("GRPCServer", query_context->getClientInfo().client_trace_context, query_context->getSettingsRef(), query_context->getOpenTelemetrySpanLog()); + thread_trace_context->root_span.kind = OpenTelemetry::SERVER; /// Prepare for sending exceptions and logs. const Settings & settings = query_context->getSettingsRef(); @@ -854,7 +855,6 @@ namespace logs_queue->max_priority = Poco::Logger::parseLevel(client_logs_level.toString()); logs_queue->setSourceRegexp(settings.send_logs_source_regexp); CurrentThread::attachInternalTextLogsQueue(logs_queue, client_logs_level); - CurrentThread::setFatalErrorCallback([this]{ onFatalError(); }); } /// Set the current database if specified. @@ -984,7 +984,10 @@ namespace executor.push(block); } - executor.finish(); + if (isQueryCancelled()) + executor.cancel(); + else + executor.finish(); } void Call::initializePipeline(const Block & header) diff --git a/src/Server/HTTP/HTMLForm.cpp b/src/Server/HTTP/HTMLForm.cpp index d9d897d20c4..1abf9e5b83e 100644 --- a/src/Server/HTTP/HTMLForm.cpp +++ b/src/Server/HTTP/HTMLForm.cpp @@ -20,6 +20,11 @@ namespace DB { +namespace ErrorCodes +{ + extern const int CANNOT_READ_ALL_DATA; +} + namespace { @@ -229,6 +234,11 @@ void HTMLForm::readMultipart(ReadBuffer & in_, PartHandler & handler) if (!in.skipToNextBoundary()) break; } + + /// It's important to check, because we could get "fake" EOF and incomplete request if a client suddenly died in the middle. + if (!in.isActualEOF()) + throw Exception(ErrorCodes::CANNOT_READ_ALL_DATA, "Unexpected EOF, " + "did not find the last boundary while parsing a multipart HTTP request"); } @@ -244,7 +254,8 @@ bool HTMLForm::MultipartReadBuffer::skipToNextBoundary() if (in.eof()) return false; - assert(boundary_hit); + chassert(boundary_hit); + chassert(!found_last_boundary); boundary_hit = false; @@ -255,7 +266,8 @@ bool HTMLForm::MultipartReadBuffer::skipToNextBoundary() { set(in.position(), 0); next(); /// We need to restrict our buffer to size of next available line. - return !startsWith(line, boundary + "--"); + found_last_boundary = startsWith(line, boundary + "--"); + return !found_last_boundary; } } diff --git a/src/Server/HTTP/HTMLForm.h b/src/Server/HTTP/HTMLForm.h index 16889b41d80..c75dafccaf0 100644 --- a/src/Server/HTTP/HTMLForm.h +++ b/src/Server/HTTP/HTMLForm.h @@ -108,10 +108,13 @@ public: /// Returns false if last boundary found. bool skipToNextBoundary(); + bool isActualEOF() const { return found_last_boundary; } + private: PeekableReadBuffer in; const std::string boundary; bool boundary_hit = true; + bool found_last_boundary = false; std::string readLine(bool append_crlf); diff --git a/src/Server/HTTP/HTTPServerRequest.cpp b/src/Server/HTTP/HTTPServerRequest.cpp index a82eb95aee1..c9ffa3a4c3b 100644 --- a/src/Server/HTTP/HTTPServerRequest.cpp +++ b/src/Server/HTTP/HTTPServerRequest.cpp @@ -12,6 +12,8 @@ #include #include +#include + #if USE_SSL #include #include @@ -44,12 +46,28 @@ HTTPServerRequest::HTTPServerRequest(HTTPContextPtr context, HTTPServerResponse readRequest(*in); /// Try parse according to RFC7230 + /// If a client crashes, most systems will gracefully terminate the connection with FIN just like it's done on close(). + /// So we will get 0 from recv(...) and will not be able to understand that something went wrong (well, we probably + /// will get RST later on attempt to write to the socket that closed on the other side, but it will happen when the query is finished). + /// If we are extremely unlucky and data format is TSV, for example, then we may stop parsing exactly between rows + /// and decide that it's EOF (but it is not). It may break deduplication, because clients cannot control it + /// and retry with exactly the same (incomplete) set of rows. + /// That's why we have to check body size if it's provided. if (getChunkedTransferEncoding()) stream = std::make_unique(std::move(in), context->getMaxChunkSize()); else if (hasContentLength()) - stream = std::make_unique(std::move(in), getContentLength(), false); + { + size_t content_length = getContentLength(); + stream = std::make_unique(std::move(in), content_length, + /* trow_exception */ true, /* exact_limit */ content_length); + } else if (getMethod() != HTTPRequest::HTTP_GET && getMethod() != HTTPRequest::HTTP_HEAD && getMethod() != HTTPRequest::HTTP_DELETE) + { stream = std::move(in); + if (!startsWith(getContentType(), "multipart/form-data")) + LOG_WARNING(&Poco::Logger::get("HTTPServerRequest"), "Got an HTTP request with no content length " + "and no chunked/multipart encoding, it may be impossible to distinguish graceful EOF from abnormal connection loss"); + } else /// We have to distinguish empty buffer and nullptr. stream = std::make_unique(); diff --git a/src/Server/HTTPHandler.cpp b/src/Server/HTTPHandler.cpp index 702743ef1f0..665b7bb7695 100644 --- a/src/Server/HTTPHandler.cpp +++ b/src/Server/HTTPHandler.cpp @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -588,11 +589,12 @@ void HTTPHandler::processQuery( /// At least, we should postpone sending of first buffer_size result bytes size_t buffer_size_total = std::max( - params.getParsed("buffer_size", DBMS_DEFAULT_BUFFER_SIZE), static_cast(DBMS_DEFAULT_BUFFER_SIZE)); + params.getParsed("buffer_size", context->getSettingsRef().http_response_buffer_size), + static_cast(DBMS_DEFAULT_BUFFER_SIZE)); /// If it is specified, the whole result will be buffered. /// First ~buffer_size bytes will be buffered in memory, the remaining bytes will be stored in temporary file. - bool buffer_until_eof = params.getParsed("wait_end_of_query", false); + bool buffer_until_eof = params.getParsed("wait_end_of_query", context->getSettingsRef().http_wait_end_of_query); size_t buffer_size_http = DBMS_DEFAULT_BUFFER_SIZE; size_t buffer_size_memory = (buffer_size_total > buffer_size_http) ? buffer_size_total : 0; @@ -678,7 +680,7 @@ void HTTPHandler::processQuery( std::unique_ptr in; static const NameSet reserved_param_names{"compress", "decompress", "user", "password", "quota_key", "query_id", "stacktrace", - "buffer_size", "wait_end_of_query", "session_id", "session_timeout", "session_check", "client_protocol_version"}; + "buffer_size", "wait_end_of_query", "session_id", "session_timeout", "session_check", "client_protocol_version", "close_session"}; Names reserved_param_suffixes; @@ -781,7 +783,6 @@ void HTTPHandler::processQuery( /// they will be applied in ProcessList::insert() from executeQuery() itself. const auto & query = getQuery(request, params, context); std::unique_ptr in_param = std::make_unique(query); - in = has_external_data ? std::move(in_param) : std::make_unique(*in_param, *in_post_maybe_compressed); /// HTTP response compression is turned on only if the client signalled that they support it /// (using Accept-Encoding header) and 'enable_http_compression' setting is turned on. @@ -831,7 +832,8 @@ void HTTPHandler::processQuery( }); } - customizeContext(request, context); + customizeContext(request, context, *in_post_maybe_compressed); + in = has_external_data ? std::move(in_param) : std::make_unique(*in_param, *in_post_maybe_compressed); executeQuery(*in, *used_output.out_maybe_delayed_and_compressed, /* allow_into_outfile = */ false, context, [&response, this] (const QueryResultDetails & details) @@ -957,6 +959,14 @@ void HTTPHandler::handleRequest(HTTPServerRequest & request, HTTPServerResponse /// In case of exception, send stack trace to client. bool with_stacktrace = false; + /// Close http session (if any) after processing the request + bool close_session = false; + String session_id; + + SCOPE_EXIT_SAFE({ + if (close_session && !session_id.empty()) + session->closeSession(session_id); + }); OpenTelemetry::TracingContextHolderPtr thread_trace_context; SCOPE_EXIT({ @@ -992,6 +1002,7 @@ void HTTPHandler::handleRequest(HTTPServerRequest & request, HTTPServerResponse client_info.client_trace_context, context->getSettingsRef(), context->getOpenTelemetrySpanLog()); + thread_trace_context->root_span.kind = OpenTelemetry::SERVER; thread_trace_context->root_span.addAttribute("clickhouse.uri", request.getURI()); response.setContentType("text/plain; charset=UTF-8"); @@ -1006,6 +1017,9 @@ void HTTPHandler::handleRequest(HTTPServerRequest & request, HTTPServerResponse HTMLForm params(default_settings, request); with_stacktrace = params.getParsed("stacktrace", false); + close_session = params.getParsed("close_session", false); + if (close_session) + session_id = params.get("session_id"); /// FIXME: maybe this check is already unnecessary. /// Workaround. Poco does not detect 411 Length Required case. @@ -1139,7 +1153,7 @@ bool PredefinedQueryHandler::customizeQueryParam(ContextMutablePtr context, cons return false; } -void PredefinedQueryHandler::customizeContext(HTTPServerRequest & request, ContextMutablePtr context) +void PredefinedQueryHandler::customizeContext(HTTPServerRequest & request, ContextMutablePtr context, ReadBuffer & body) { /// If in the configuration file, the handler's header is regex and contains named capture group /// We will extract regex named capture groups as query parameters @@ -1173,6 +1187,15 @@ void PredefinedQueryHandler::customizeContext(HTTPServerRequest & request, Conte const auto & header_value = request.get(header_name); set_query_params(header_value.data(), header_value.data() + header_value.size(), regex); } + + if (unlikely(receive_params.contains("_request_body") && !context->getQueryParameters().contains("_request_body"))) + { + WriteBufferFromOwnString value; + const auto & settings = context->getSettingsRef(); + + copyDataMaxBytes(body, value, settings.http_max_request_param_data_size); + context->setQueryParameter("_request_body", value.str()); + } } std::string PredefinedQueryHandler::getQuery(HTTPServerRequest & request, HTMLForm & params, ContextMutablePtr context) diff --git a/src/Server/HTTPHandler.h b/src/Server/HTTPHandler.h index fa742ebc8fb..5eda5927538 100644 --- a/src/Server/HTTPHandler.h +++ b/src/Server/HTTPHandler.h @@ -36,7 +36,7 @@ public: void handleRequest(HTTPServerRequest & request, HTTPServerResponse & response) override; /// This method is called right before the query execution. - virtual void customizeContext(HTTPServerRequest & /* request */, ContextMutablePtr /* context */) {} + virtual void customizeContext(HTTPServerRequest & /* request */, ContextMutablePtr /* context */, ReadBuffer & /* body */) {} virtual bool customizeQueryParam(ContextMutablePtr context, const std::string & key, const std::string & value) = 0; @@ -163,7 +163,7 @@ public: , const CompiledRegexPtr & url_regex_, const std::unordered_map & header_name_with_regex_ , const std::optional & content_type_override_); - virtual void customizeContext(HTTPServerRequest & request, ContextMutablePtr context) override; + void customizeContext(HTTPServerRequest & request, ContextMutablePtr context, ReadBuffer & body) override; std::string getQuery(HTTPServerRequest & request, HTMLForm & params, ContextMutablePtr context) override; diff --git a/src/Server/KeeperTCPHandler.cpp b/src/Server/KeeperTCPHandler.cpp index 0853c6ee62b..f9e11062906 100644 --- a/src/Server/KeeperTCPHandler.cpp +++ b/src/Server/KeeperTCPHandler.cpp @@ -20,7 +20,7 @@ #include #include #include -#include +#include #ifdef POCO_HAVE_FD_EPOLL diff --git a/src/Server/MySQLHandler.cpp b/src/Server/MySQLHandler.cpp index d8ea359ce5f..40cc51f8aae 100644 --- a/src/Server/MySQLHandler.cpp +++ b/src/Server/MySQLHandler.cpp @@ -155,7 +155,7 @@ void MySQLHandler::run() payload.readStrict(command); // For commands which are executed without MemoryTracker. - LimitReadBuffer limited_payload(payload, 10000, true, "too long MySQL packet."); + LimitReadBuffer limited_payload(payload, 10000, /* trow_exception */ true, /* exact_limit */ {}, "too long MySQL packet."); LOG_DEBUG(log, "Received command: {}. Connection id: {}.", static_cast(static_cast(command)), connection_id); diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index a307b472a64..cc050730853 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -41,6 +41,7 @@ #include #include #include +#include #include #include @@ -109,6 +110,7 @@ namespace ErrorCodes extern const int UNEXPECTED_PACKET_FROM_CLIENT; extern const int UNKNOWN_PROTOCOL; extern const int AUTHENTICATION_FAILED; + extern const int QUERY_WAS_CANCELLED; } TCPHandler::TCPHandler(IServer & server_, TCPServer & tcp_server_, const Poco::Net::StreamSocket & socket_, bool parse_proxy_protocol_, std::string server_display_name_) @@ -276,8 +278,13 @@ void TCPHandler::runImpl() query_context->getClientInfo().client_trace_context, query_context->getSettingsRef(), query_context->getOpenTelemetrySpanLog()); + thread_trace_context->root_span.kind = OpenTelemetry::SERVER; - query_scope.emplace(query_context); + query_scope.emplace(query_context, /* fatal_error_callback */ [this] + { + std::lock_guard lock(fatal_error_mutex); + sendLogs(); + }); /// If query received, then settings in query_context has been updated. /// So it's better to update the connection settings for flexibility. @@ -298,11 +305,6 @@ void TCPHandler::runImpl() state.logs_queue->max_priority = Poco::Logger::parseLevel(client_logs_level.toString()); state.logs_queue->setSourceRegexp(query_context->getSettingsRef().send_logs_source_regexp); CurrentThread::attachInternalTextLogsQueue(state.logs_queue, client_logs_level); - CurrentThread::setFatalErrorCallback([this] - { - std::lock_guard lock(fatal_error_mutex); - sendLogs(); - }); } if (client_tcp_protocol_version >= DBMS_MIN_PROTOCOL_VERSION_WITH_INCREMENTAL_PROFILE_EVENTS) { @@ -415,17 +417,25 @@ void TCPHandler::runImpl() after_check_cancelled.restart(); after_send_progress.restart(); + auto finish_or_cancel = [this]() + { + if (state.is_cancelled) + state.io.onCancelOrConnectionLoss(); + else + state.io.onFinish(); + }; + if (state.io.pipeline.pushing()) { /// FIXME: check explicitly that insert query suggests to receive data via native protocol, state.need_receive_data_for_insert = true; processInsertQuery(); - state.io.onFinish(); + finish_or_cancel(); } else if (state.io.pipeline.pulling()) { processOrdinaryQueryWithProcessors(); - state.io.onFinish(); + finish_or_cancel(); } else if (state.io.pipeline.completed()) { @@ -454,7 +464,7 @@ void TCPHandler::runImpl() executor.execute(); } - state.io.onFinish(); + finish_or_cancel(); std::lock_guard lock(task_callback_mutex); @@ -468,7 +478,7 @@ void TCPHandler::runImpl() } else { - state.io.onFinish(); + finish_or_cancel(); } /// Do it before sending end of stream, to have a chance to show log message in client. @@ -611,8 +621,6 @@ void TCPHandler::runImpl() /// It is important to destroy query context here. We do not want it to live arbitrarily longer than the query. query_context.reset(); - CurrentThread::setFatalErrorCallback({}); - if (is_interserver_mode) { /// We don't really have session in interserver mode, new one is created for each query. It's better to reset it now. @@ -636,7 +644,6 @@ void TCPHandler::extractConnectionSettingsFromContext(const ContextPtr & context interactive_delay = settings.interactive_delay; sleep_in_send_tables_status = settings.sleep_in_send_tables_status_ms; unknown_packet_in_send_data = settings.unknown_packet_in_send_data; - sleep_in_receive_cancel = settings.sleep_in_receive_cancel_ms; sleep_after_receiving_query = settings.sleep_after_receiving_query_ms; } @@ -660,6 +667,7 @@ bool TCPHandler::readDataNext() { LOG_INFO(log, "Client has dropped the connection, cancel the query."); state.is_connection_closed = true; + state.is_cancelled = true; break; } @@ -703,6 +711,9 @@ void TCPHandler::readData() while (readDataNext()) ; + + if (state.is_cancelled) + throw Exception(ErrorCodes::QUERY_WAS_CANCELLED, "Query was cancelled"); } @@ -713,6 +724,9 @@ void TCPHandler::skipData() while (readDataNext()) ; + + if (state.is_cancelled) + throw Exception(ErrorCodes::QUERY_WAS_CANCELLED, "Query was cancelled"); } @@ -749,7 +763,10 @@ void TCPHandler::processInsertQuery() while (readDataNext()) executor.push(std::move(state.block_for_insert)); - executor.finish(); + if (state.is_cancelled) + executor.cancel(); + else + executor.finish(); }; if (num_threads > 1) @@ -1048,7 +1065,7 @@ bool TCPHandler::receiveProxyHeader() /// Only PROXYv1 is supported. /// Validation of protocol is not fully performed. - LimitReadBuffer limit_in(*in, 107, true); /// Maximum length from the specs. + LimitReadBuffer limit_in(*in, 107, /* trow_exception */ true, /* exact_limit */ {}); /// Maximum length from the specs. assertString("PROXY ", limit_in); @@ -1208,14 +1225,7 @@ void TCPHandler::receiveHello() session = makeSession(); auto & client_info = session->getClientInfo(); - - /// Extract the last entry from comma separated list of forwarded_for addresses. - /// Only the last proxy can be trusted (if any). - String forwarded_address = client_info.getLastForwardedFor(); - if (!forwarded_address.empty() && server.config().getBool("auth_use_forwarded_address", false)) - session->authenticate(user, password, Poco::Net::SocketAddress(forwarded_address, socket().peerAddress().port())); - else - session->authenticate(user, password, socket().peerAddress()); + session->authenticate(user, password, getClientAddress(client_info)); } void TCPHandler::receiveAddendum() @@ -1270,6 +1280,18 @@ void TCPHandler::sendHello() writeStringBinary(exception_message, *out); } } + if (client_tcp_protocol_version >= DBMS_MIN_REVISION_WITH_INTERSERVER_SECRET_V2) + { + chassert(!nonce.has_value()); + /// Contains lots of stuff (including time), so this should be enough for NONCE. + nonce.emplace(thread_local_rng()); + writeIntBinary(nonce.value(), *out); + } + else + { + LOG_WARNING(LogFrequencyLimiter(log, 10), + "Using deprecated interserver protocol because the client is too old. Consider upgrading all nodes in cluster."); + } out->next(); } @@ -1308,16 +1330,9 @@ bool TCPHandler::receivePacket() return false; case Protocol::Client::Cancel: - { - /// For testing connection collector. - if (unlikely(sleep_in_receive_cancel.totalMilliseconds())) - { - std::chrono::milliseconds ms(sleep_in_receive_cancel.totalMilliseconds()); - std::this_thread::sleep_for(ms); - } - + LOG_INFO(log, "Received 'Cancel' packet from the client, canceling the query"); + state.is_cancelled = true; return false; - } case Protocol::Client::Hello: receiveUnexpectedHello(); @@ -1357,13 +1372,8 @@ String TCPHandler::receiveReadTaskResponseAssumeLocked() { if (packet_type == Protocol::Client::Cancel) { + LOG_INFO(log, "Received 'Cancel' packet from the client, canceling the read task"); state.is_cancelled = true; - /// For testing connection collector. - if (unlikely(sleep_in_receive_cancel.totalMilliseconds())) - { - std::chrono::milliseconds ms(sleep_in_receive_cancel.totalMilliseconds()); - std::this_thread::sleep_for(ms); - } return {}; } else @@ -1390,13 +1400,8 @@ std::optional TCPHandler::receivePartitionMergeTreeReadTas { if (packet_type == Protocol::Client::Cancel) { + LOG_INFO(log, "Received 'Cancel' packet from the client, canceling the MergeTree read task"); state.is_cancelled = true; - /// For testing connection collector. - if (unlikely(sleep_in_receive_cancel.totalMilliseconds())) - { - std::chrono::milliseconds ms(sleep_in_receive_cancel.totalMilliseconds()); - std::this_thread::sleep_for(ms); - } return std::nullopt; } else @@ -1467,20 +1472,30 @@ void TCPHandler::receiveQuery() if (client_tcp_protocol_version >= DBMS_MIN_PROTOCOL_VERSION_WITH_PARAMETERS) passed_params.read(*in, settings_format); - /// TODO Unify interserver authentication (and make sure that it's secure enough) if (is_interserver_mode) { client_info.interface = ClientInfo::Interface::TCP_INTERSERVER; #if USE_SSL String cluster_secret = server.context()->getCluster(cluster)->getSecret(); + if (salt.empty() || cluster_secret.empty()) { - auto exception = Exception(ErrorCodes::AUTHENTICATION_FAILED, "Interserver authentication failed"); - session->onAuthenticationFailure(/* user_name */ std::nullopt, socket().peerAddress(), exception); + auto exception = Exception(ErrorCodes::AUTHENTICATION_FAILED, "Interserver authentication failed (no salt/cluster secret)"); + session->onAuthenticationFailure(/* user_name= */ std::nullopt, socket().peerAddress(), exception); + throw exception; /// NOLINT + } + + if (client_tcp_protocol_version >= DBMS_MIN_REVISION_WITH_INTERSERVER_SECRET_V2 && !nonce.has_value()) + { + auto exception = Exception(ErrorCodes::AUTHENTICATION_FAILED, "Interserver authentication failed (no nonce)"); + session->onAuthenticationFailure(/* user_name= */ std::nullopt, socket().peerAddress(), exception); throw exception; /// NOLINT } std::string data(salt); + // For backward compatibility + if (nonce.has_value()) + data += std::to_string(nonce.value()); data += cluster_secret; data += state.query; data += state.query_id; @@ -1501,11 +1516,16 @@ void TCPHandler::receiveQuery() /// so we should not rely on that. However, in this particular case we got client_info from other clickhouse-server, so it's ok. if (client_info.initial_user.empty()) { - LOG_DEBUG(log, "User (no user, interserver mode)"); + LOG_DEBUG(log, "User (no user, interserver mode) (client: {})", getClientAddress(client_info).toString()); } else { - LOG_DEBUG(log, "User (initial, interserver mode): {}", client_info.initial_user); + LOG_DEBUG(log, "User (initial, interserver mode): {} (client: {})", client_info.initial_user, getClientAddress(client_info).toString()); + /// In case of inter-server mode authorization is done with the + /// initial address of the client, not the real address from which + /// the query was come, since the real address is the address of + /// the initiator server, while we are interested in client's + /// address. session->authenticate(AlwaysAllowCredentials{client_info.initial_user}, client_info.initial_address); } #else @@ -1802,15 +1822,6 @@ bool TCPHandler::isQueryCancelled() throw NetException(ErrorCodes::UNEXPECTED_PACKET_FROM_CLIENT, "Unexpected packet Cancel received from client"); LOG_INFO(log, "Query was cancelled."); state.is_cancelled = true; - /// For testing connection collector. - { - if (unlikely(sleep_in_receive_cancel.totalMilliseconds())) - { - std::chrono::milliseconds ms(sleep_in_receive_cancel.totalMilliseconds()); - std::this_thread::sleep_for(ms); - } - } - return true; default: @@ -1991,4 +2002,15 @@ void TCPHandler::run() } } +Poco::Net::SocketAddress TCPHandler::getClientAddress(const ClientInfo & client_info) +{ + /// Extract the last entry from comma separated list of forwarded_for addresses. + /// Only the last proxy can be trusted (if any). + String forwarded_address = client_info.getLastForwardedFor(); + if (!forwarded_address.empty() && server.config().getBool("auth_use_forwarded_address", false)) + return Poco::Net::SocketAddress(forwarded_address, socket().peerAddress().port()); + else + return socket().peerAddress(); +} + } diff --git a/src/Server/TCPHandler.h b/src/Server/TCPHandler.h index f06b0b060b3..d5e14295dc3 100644 --- a/src/Server/TCPHandler.h +++ b/src/Server/TCPHandler.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include @@ -170,7 +171,6 @@ private: UInt64 interactive_delay = 100000; Poco::Timespan sleep_in_send_tables_status; UInt64 unknown_packet_in_send_data = 0; - Poco::Timespan sleep_in_receive_cancel; Poco::Timespan sleep_after_receiving_query; std::unique_ptr session; @@ -189,7 +189,10 @@ private: /// For inter-server secret (remote_server.*.secret) bool is_interserver_mode = false; + /// For DBMS_MIN_REVISION_WITH_INTERSERVER_SECRET String salt; + /// For DBMS_MIN_REVISION_WITH_INTERSERVER_SECRET_V2 + std::optional nonce; String cluster; std::mutex task_callback_mutex; @@ -273,6 +276,8 @@ private: /// This function is called from different threads. void updateProgress(const Progress & value); + + Poco::Net::SocketAddress getClientAddress(const ClientInfo & client_info); }; } diff --git a/src/Storages/AlterCommands.cpp b/src/Storages/AlterCommands.cpp index da11a87eb4d..aff17465466 100644 --- a/src/Storages/AlterCommands.cpp +++ b/src/Storages/AlterCommands.cpp @@ -719,7 +719,7 @@ bool isMetadataOnlyConversion(const IDataType * from, const IDataType * to) { typeid(DataTypeUInt16), typeid(DataTypeDate) }, }; - /// Unwrap some nested and check for valid conevrsions + /// Unwrap some nested and check for valid conversions while (true) { /// types are equal, obviously pure metadata alter @@ -749,10 +749,9 @@ bool isMetadataOnlyConversion(const IDataType * from, const IDataType * to) const auto * nullable_from = typeid_cast(from); const auto * nullable_to = typeid_cast(to); - if (nullable_to) + if (nullable_from && nullable_to) { - /// Here we allow a conversion X -> Nullable(X) to make a metadata-only conversion. - from = nullable_from ? nullable_from->getNestedType().get() : from; + from = nullable_from->getNestedType().get(); to = nullable_to->getNestedType().get(); continue; } diff --git a/src/Storages/Cache/ExternalDataSourceCache.cpp b/src/Storages/Cache/ExternalDataSourceCache.cpp index 56b2e661836..1fc68a2d774 100644 --- a/src/Storages/Cache/ExternalDataSourceCache.cpp +++ b/src/Storages/Cache/ExternalDataSourceCache.cpp @@ -15,7 +15,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/src/Storages/ColumnsDescription.cpp b/src/Storages/ColumnsDescription.cpp index d401840eec7..fa39e304925 100644 --- a/src/Storages/ColumnsDescription.cpp +++ b/src/Storages/ColumnsDescription.cpp @@ -383,6 +383,15 @@ NamesAndTypesList ColumnsDescription::getEphemeral() const return ret; } +NamesAndTypesList ColumnsDescription::getWithDefaultExpression() const +{ + NamesAndTypesList ret; + for (const auto & col : columns) + if (col.default_desc.expression) + ret.emplace_back(col.name, col.type); + return ret; +} + NamesAndTypesList ColumnsDescription::getAll() const { NamesAndTypesList ret; diff --git a/src/Storages/ColumnsDescription.h b/src/Storages/ColumnsDescription.h index 4f874f4b850..5551fdea2e3 100644 --- a/src/Storages/ColumnsDescription.h +++ b/src/Storages/ColumnsDescription.h @@ -132,6 +132,7 @@ public: NamesAndTypesList getInsertable() const; /// ordinary + ephemeral NamesAndTypesList getAliases() const; NamesAndTypesList getEphemeral() const; + NamesAndTypesList getWithDefaultExpression() const; // columns with default expression, for example set by `CREATE TABLE` statement NamesAndTypesList getAllPhysical() const; /// ordinary + materialized. NamesAndTypesList getAll() const; /// ordinary + materialized + aliases + ephemeral /// Returns .size0/.null/... diff --git a/src/Storages/Distributed/DirectoryMonitor.cpp b/src/Storages/Distributed/DirectoryMonitor.cpp deleted file mode 100644 index cb6659e59ce..00000000000 --- a/src/Storages/Distributed/DirectoryMonitor.cpp +++ /dev/null @@ -1,1239 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - - -namespace CurrentMetrics -{ - extern const Metric DistributedSend; - extern const Metric DistributedFilesToInsert; - extern const Metric BrokenDistributedFilesToInsert; -} - -namespace fs = std::filesystem; - -namespace DB -{ - -namespace ErrorCodes -{ - extern const int CANNOT_READ_ALL_DATA; - extern const int UNKNOWN_CODEC; - extern const int CANNOT_DECOMPRESS; - extern const int CHECKSUM_DOESNT_MATCH; - extern const int TOO_LARGE_SIZE_COMPRESSED; - extern const int ATTEMPT_TO_READ_AFTER_EOF; - extern const int EMPTY_DATA_PASSED; - extern const int INCORRECT_FILE_NAME; - extern const int MEMORY_LIMIT_EXCEEDED; - extern const int DISTRIBUTED_BROKEN_BATCH_INFO; - extern const int DISTRIBUTED_BROKEN_BATCH_FILES; - extern const int TOO_MANY_PARTS; - extern const int TOO_MANY_BYTES; - extern const int TOO_MANY_ROWS_OR_BYTES; - extern const int TOO_MANY_PARTITIONS; - extern const int DISTRIBUTED_TOO_MANY_PENDING_BYTES; - extern const int ARGUMENT_OUT_OF_BOUND; -} - - -namespace -{ - constexpr const std::chrono::minutes decrease_error_count_period{5}; - - template - ConnectionPoolPtrs createPoolsForAddresses(const std::string & name, PoolFactory && factory, const Cluster::ShardsInfo & shards_info, Poco::Logger * log) - { - ConnectionPoolPtrs pools; - - auto make_connection = [&](const Cluster::Address & address) - { - try - { - pools.emplace_back(factory(address)); - } - catch (const Exception & e) - { - if (e.code() == ErrorCodes::INCORRECT_FILE_NAME) - { - tryLogCurrentException(log); - return; - } - throw; - } - }; - - for (auto it = boost::make_split_iterator(name, boost::first_finder(",")); it != decltype(it){}; ++it) - { - const std::string & dirname = boost::copy_range(*it); - Cluster::Address address = Cluster::Address::fromFullString(dirname); - if (address.shard_index && dirname.ends_with("_all_replicas")) - { - if (address.shard_index > shards_info.size()) - { - LOG_ERROR(log, "No shard with shard_index={} ({})", address.shard_index, name); - continue; - } - - const auto & shard_info = shards_info[address.shard_index - 1]; - size_t replicas = shard_info.per_replica_pools.size(); - - for (size_t replica_index = 1; replica_index <= replicas; ++replica_index) - { - address.replica_index = static_cast(replica_index); - make_connection(address); - } - } - else - make_connection(address); - } - - return pools; - } - - void assertChecksum(CityHash_v1_0_2::uint128 expected, CityHash_v1_0_2::uint128 calculated) - { - if (expected != calculated) - { - throw Exception(ErrorCodes::CHECKSUM_DOESNT_MATCH, - "Checksum of extra info doesn't match: corrupted data. Reference: {}{}. Actual: {}{}.", - getHexUIntLowercase(expected.first), getHexUIntLowercase(expected.second), - getHexUIntLowercase(calculated.first), getHexUIntLowercase(calculated.second)); - } - } - - struct DistributedHeader - { - UInt64 revision = 0; - Settings insert_settings; - std::string insert_query; - ClientInfo client_info; - - /// .bin file cannot have zero rows/bytes. - size_t rows = 0; - size_t bytes = 0; - - UInt32 shard_num = 0; - std::string cluster; - std::string distributed_table; - std::string remote_table; - - /// dumpStructure() of the header -- obsolete - std::string block_header_string; - Block block_header; - }; - - DistributedHeader readDistributedHeader(ReadBufferFromFile & in, Poco::Logger * log) - { - DistributedHeader distributed_header; - - UInt64 query_size; - readVarUInt(query_size, in); - - if (query_size == DBMS_DISTRIBUTED_SIGNATURE_HEADER) - { - /// Read the header as a string. - String header_data; - readStringBinary(header_data, in); - - /// Check the checksum of the header. - CityHash_v1_0_2::uint128 checksum; - readPODBinary(checksum, in); - assertChecksum(checksum, CityHash_v1_0_2::CityHash128(header_data.data(), header_data.size())); - - /// Read the parts of the header. - ReadBufferFromString header_buf(header_data); - - readVarUInt(distributed_header.revision, header_buf); - if (DBMS_TCP_PROTOCOL_VERSION < distributed_header.revision) - { - LOG_WARNING(log, "ClickHouse shard version is older than ClickHouse initiator version. It may lack support for new features."); - } - - readStringBinary(distributed_header.insert_query, header_buf); - distributed_header.insert_settings.read(header_buf); - - if (header_buf.hasPendingData()) - distributed_header.client_info.read(header_buf, distributed_header.revision); - - if (header_buf.hasPendingData()) - { - readVarUInt(distributed_header.rows, header_buf); - readVarUInt(distributed_header.bytes, header_buf); - readStringBinary(distributed_header.block_header_string, header_buf); - } - - if (header_buf.hasPendingData()) - { - NativeReader header_block_in(header_buf, distributed_header.revision); - distributed_header.block_header = header_block_in.read(); - if (!distributed_header.block_header) - throw Exception(ErrorCodes::CANNOT_READ_ALL_DATA, - "Cannot read header from the {} batch. Data was written with protocol version {}, current version: {}", - in.getFileName(), distributed_header.revision, DBMS_TCP_PROTOCOL_VERSION); - } - - if (header_buf.hasPendingData()) - { - readVarUInt(distributed_header.shard_num, header_buf); - readStringBinary(distributed_header.cluster, header_buf); - readStringBinary(distributed_header.distributed_table, header_buf); - readStringBinary(distributed_header.remote_table, header_buf); - } - - /// Add handling new data here, for example: - /// - /// if (header_buf.hasPendingData()) - /// readVarUInt(my_new_data, header_buf); - /// - /// And note that it is safe, because we have checksum and size for header. - - return distributed_header; - } - - if (query_size == DBMS_DISTRIBUTED_SIGNATURE_HEADER_OLD_FORMAT) - { - distributed_header.insert_settings.read(in, SettingsWriteFormat::BINARY); - readStringBinary(distributed_header.insert_query, in); - return distributed_header; - } - - distributed_header.insert_query.resize(query_size); - in.readStrict(distributed_header.insert_query.data(), query_size); - - return distributed_header; - } - - /// 'remote_error' argument is used to decide whether some errors should be - /// ignored or not, in particular: - /// - /// - ATTEMPT_TO_READ_AFTER_EOF should not be ignored - /// if we receive it from remote (receiver), since: - /// - the sender will got ATTEMPT_TO_READ_AFTER_EOF when the client just go away, - /// i.e. server had been restarted - /// - since #18853 the file will be checked on the sender locally, and - /// if there is something wrong with the file itself, we will receive - /// ATTEMPT_TO_READ_AFTER_EOF not from the remote at first - /// and mark batch as broken. - bool isFileBrokenErrorCode(int code, bool remote_error) - { - return code == ErrorCodes::CHECKSUM_DOESNT_MATCH - || code == ErrorCodes::EMPTY_DATA_PASSED - || code == ErrorCodes::TOO_LARGE_SIZE_COMPRESSED - || code == ErrorCodes::CANNOT_READ_ALL_DATA - || code == ErrorCodes::UNKNOWN_CODEC - || code == ErrorCodes::CANNOT_DECOMPRESS - || code == ErrorCodes::DISTRIBUTED_BROKEN_BATCH_INFO - || code == ErrorCodes::DISTRIBUTED_BROKEN_BATCH_FILES - || (!remote_error && code == ErrorCodes::ATTEMPT_TO_READ_AFTER_EOF); - } - - /// Can the batch be split and send files from batch one-by-one instead? - bool isSplittableErrorCode(int code, bool remote) - { - return code == ErrorCodes::MEMORY_LIMIT_EXCEEDED - /// FunctionRange::max_elements and similar - || code == ErrorCodes::ARGUMENT_OUT_OF_BOUND - || code == ErrorCodes::TOO_MANY_PARTS - || code == ErrorCodes::TOO_MANY_BYTES - || code == ErrorCodes::TOO_MANY_ROWS_OR_BYTES - || code == ErrorCodes::TOO_MANY_PARTITIONS - || code == ErrorCodes::DISTRIBUTED_TOO_MANY_PENDING_BYTES - || code == ErrorCodes::DISTRIBUTED_BROKEN_BATCH_INFO - || isFileBrokenErrorCode(code, remote) - ; - } - - SyncGuardPtr getDirectorySyncGuard(bool dir_fsync, const DiskPtr & disk, const String & path) - { - if (dir_fsync) - return disk->getDirectorySyncGuard(path); - return nullptr; - } - - void writeAndConvert(RemoteInserter & remote, const DistributedHeader & distributed_header, ReadBufferFromFile & in) - { - CompressedReadBuffer decompressing_in(in); - NativeReader block_in(decompressing_in, distributed_header.revision); - - while (Block block = block_in.read()) - { - auto converting_dag = ActionsDAG::makeConvertingActions( - block.cloneEmpty().getColumnsWithTypeAndName(), - remote.getHeader().getColumnsWithTypeAndName(), - ActionsDAG::MatchColumnsMode::Name); - - auto converting_actions = std::make_shared(std::move(converting_dag)); - converting_actions->execute(block); - remote.write(block); - } - } - - void writeRemoteConvert( - const DistributedHeader & distributed_header, - RemoteInserter & remote, - bool compression_expected, - ReadBufferFromFile & in, - Poco::Logger * log) - { - if (!remote.getHeader()) - { - CheckingCompressedReadBuffer checking_in(in); - remote.writePrepared(checking_in); - return; - } - - /// This is old format, that does not have header for the block in the file header, - /// applying ConvertingTransform in this case is not a big overhead. - /// - /// Anyway we can get header only from the first block, which contain all rows anyway. - if (!distributed_header.block_header) - { - LOG_TRACE(log, "Processing batch {} with old format (no header)", in.getFileName()); - - writeAndConvert(remote, distributed_header, in); - return; - } - - if (!blocksHaveEqualStructure(distributed_header.block_header, remote.getHeader())) - { - LOG_WARNING(log, - "Structure does not match (remote: {}, local: {}), implicit conversion will be done", - remote.getHeader().dumpStructure(), distributed_header.block_header.dumpStructure()); - - writeAndConvert(remote, distributed_header, in); - return; - } - - /// If connection does not use compression, we have to uncompress the data. - if (!compression_expected) - { - writeAndConvert(remote, distributed_header, in); - return; - } - - if (distributed_header.revision != remote.getServerRevision()) - { - writeAndConvert(remote, distributed_header, in); - return; - } - - /// Otherwise write data as it was already prepared (more efficient path). - CheckingCompressedReadBuffer checking_in(in); - remote.writePrepared(checking_in); - } - - uint64_t doubleToUInt64(double d) - { - if (d >= static_cast(std::numeric_limits::max())) - return std::numeric_limits::max(); - return static_cast(d); - } -} - - -StorageDistributedDirectoryMonitor::StorageDistributedDirectoryMonitor( - StorageDistributed & storage_, - const DiskPtr & disk_, - const std::string & relative_path_, - ConnectionPoolPtr pool_, - ActionBlocker & monitor_blocker_, - BackgroundSchedulePool & bg_pool) - : storage(storage_) - , pool(std::move(pool_)) - , disk(disk_) - , relative_path(relative_path_) - , path(fs::path(disk->getPath()) / relative_path / "") - , should_batch_inserts(storage.getDistributedSettingsRef().monitor_batch_inserts) - , split_batch_on_failure(storage.getDistributedSettingsRef().monitor_split_batch_on_failure) - , dir_fsync(storage.getDistributedSettingsRef().fsync_directories) - , min_batched_block_size_rows(storage.getContext()->getSettingsRef().min_insert_block_size_rows) - , min_batched_block_size_bytes(storage.getContext()->getSettingsRef().min_insert_block_size_bytes) - , current_batch_file_path(path + "current_batch.txt") - , default_sleep_time(storage.getDistributedSettingsRef().monitor_sleep_time_ms.totalMilliseconds()) - , sleep_time(default_sleep_time) - , max_sleep_time(storage.getDistributedSettingsRef().monitor_max_sleep_time_ms.totalMilliseconds()) - , log(&Poco::Logger::get(getLoggerName())) - , monitor_blocker(monitor_blocker_) - , metric_pending_files(CurrentMetrics::DistributedFilesToInsert, 0) - , metric_broken_files(CurrentMetrics::BrokenDistributedFilesToInsert, 0) -{ - task_handle = bg_pool.createTask(getLoggerName() + "/Bg", [this]{ run(); }); - task_handle->activateAndSchedule(); -} - - -StorageDistributedDirectoryMonitor::~StorageDistributedDirectoryMonitor() -{ - if (!quit) - { - quit = true; - task_handle->deactivate(); - } -} - -void StorageDistributedDirectoryMonitor::flushAllData() -{ - if (quit) - return; - - std::lock_guard lock{mutex}; - - const auto & files = getFiles(); - if (!files.empty()) - { - processFiles(files); - - /// Update counters. - getFiles(); - } -} - -void StorageDistributedDirectoryMonitor::shutdownAndDropAllData() -{ - if (!quit) - { - quit = true; - task_handle->deactivate(); - } - - auto dir_sync_guard = getDirectorySyncGuard(dir_fsync, disk, relative_path); - fs::remove_all(path); -} - - -void StorageDistributedDirectoryMonitor::run() -{ - std::lock_guard lock{mutex}; - - bool do_sleep = false; - while (!quit) - { - do_sleep = true; - - const auto & files = getFiles(); - if (files.empty()) - break; - - if (!monitor_blocker.isCancelled()) - { - try - { - do_sleep = !processFiles(files); - - std::lock_guard status_lock(status_mutex); - status.last_exception = std::exception_ptr{}; - } - catch (...) - { - std::lock_guard status_lock(status_mutex); - - do_sleep = true; - ++status.error_count; - - UInt64 q = doubleToUInt64(std::exp2(status.error_count)); - std::chrono::milliseconds new_sleep_time(default_sleep_time.count() * q); - if (new_sleep_time.count() < 0) - sleep_time = max_sleep_time; - else - sleep_time = std::min(new_sleep_time, max_sleep_time); - - tryLogCurrentException(getLoggerName().data()); - status.last_exception = std::current_exception(); - status.last_exception_time = std::chrono::system_clock::now(); - } - } - else - { - LOG_DEBUG(log, "Skipping send data over distributed table."); - } - - const auto now = std::chrono::system_clock::now(); - if (now - last_decrease_time > decrease_error_count_period) - { - std::lock_guard status_lock(status_mutex); - - status.error_count /= 2; - last_decrease_time = now; - } - - if (do_sleep) - break; - } - - /// Update counters. - getFiles(); - - if (!quit && do_sleep) - task_handle->scheduleAfter(sleep_time.count()); -} - - -ConnectionPoolPtr StorageDistributedDirectoryMonitor::createPool(const std::string & name, const StorageDistributed & storage) -{ - const auto pool_factory = [&storage, &name] (const Cluster::Address & address) -> ConnectionPoolPtr - { - const auto & cluster = storage.getCluster(); - const auto & shards_info = cluster->getShardsInfo(); - const auto & shards_addresses = cluster->getShardsAddresses(); - - /// Check new format shard{shard_index}_replica{replica_index} - /// (shard_index and replica_index starts from 1). - if (address.shard_index != 0) - { - if (!address.replica_index) - throw Exception(ErrorCodes::INCORRECT_FILE_NAME, - "Wrong replica_index={} ({})", address.replica_index, name); - - if (address.shard_index > shards_info.size()) - throw Exception(ErrorCodes::INCORRECT_FILE_NAME, - "No shard with shard_index={} ({})", address.shard_index, name); - - const auto & shard_info = shards_info[address.shard_index - 1]; - if (address.replica_index > shard_info.per_replica_pools.size()) - throw Exception(ErrorCodes::INCORRECT_FILE_NAME, - "No shard with replica_index={} ({})", address.replica_index, name); - - return shard_info.per_replica_pools[address.replica_index - 1]; - } - - /// Existing connections pool have a higher priority. - for (size_t shard_index = 0; shard_index < shards_info.size(); ++shard_index) - { - const Cluster::Addresses & replicas_addresses = shards_addresses[shard_index]; - - for (size_t replica_index = 0; replica_index < replicas_addresses.size(); ++replica_index) - { - const Cluster::Address & replica_address = replicas_addresses[replica_index]; - - if (address.user == replica_address.user && - address.password == replica_address.password && - address.host_name == replica_address.host_name && - address.port == replica_address.port && - address.default_database == replica_address.default_database && - address.secure == replica_address.secure) - { - return shards_info[shard_index].per_replica_pools[replica_index]; - } - } - } - - return std::make_shared( - 1, /* max_connections */ - address.host_name, - address.port, - address.default_database, - address.user, - address.password, - address.quota_key, - address.cluster, - address.cluster_secret, - storage.getName() + '_' + address.user, /* client */ - Protocol::Compression::Enable, - address.secure); - }; - - auto pools = createPoolsForAddresses(name, pool_factory, storage.getCluster()->getShardsInfo(), storage.log); - - const auto settings = storage.getContext()->getSettings(); - return pools.size() == 1 ? pools.front() : std::make_shared(pools, - settings.load_balancing, - settings.distributed_replica_error_half_life.totalSeconds(), - settings.distributed_replica_error_cap); -} - - -std::map StorageDistributedDirectoryMonitor::getFiles() -{ - std::map files; - - fs::directory_iterator end; - for (fs::directory_iterator it{path}; it != end; ++it) - { - const auto & file_path_str = it->path(); - if (!it->is_directory() && startsWith(fs::path(file_path_str).extension(), ".bin")) - { - files[parse(fs::path(file_path_str).stem())] = file_path_str; - } - } - - return files; -} -bool StorageDistributedDirectoryMonitor::processFiles(const std::map & files) -{ - if (should_batch_inserts) - { - processFilesWithBatching(files); - } - else - { - for (const auto & file : files) - { - if (quit) - return true; - - processFile(file.second); - } - } - - return true; -} - -void StorageDistributedDirectoryMonitor::processFile(const std::string & file_path) -{ - OpenTelemetry::TracingContextHolderPtr thread_trace_context; - - Stopwatch watch; - try - { - CurrentMetrics::Increment metric_increment{CurrentMetrics::DistributedSend}; - - ReadBufferFromFile in(file_path); - const auto & distributed_header = readDistributedHeader(in, log); - - thread_trace_context = std::make_unique(__PRETTY_FUNCTION__, - distributed_header.client_info.client_trace_context, - this->storage.getContext()->getOpenTelemetrySpanLog()); - thread_trace_context->root_span.addAttribute("clickhouse.shard_num", distributed_header.shard_num); - thread_trace_context->root_span.addAttribute("clickhouse.cluster", distributed_header.cluster); - thread_trace_context->root_span.addAttribute("clickhouse.distributed", distributed_header.distributed_table); - thread_trace_context->root_span.addAttribute("clickhouse.remote", distributed_header.remote_table); - thread_trace_context->root_span.addAttribute("clickhouse.rows", distributed_header.rows); - thread_trace_context->root_span.addAttribute("clickhouse.bytes", distributed_header.bytes); - - auto timeouts = ConnectionTimeouts::getTCPTimeoutsWithFailover(distributed_header.insert_settings); - auto connection = pool->get(timeouts, &distributed_header.insert_settings); - LOG_DEBUG(log, "Sending `{}` to {} ({} rows, {} bytes)", - file_path, - connection->getDescription(), - formatReadableQuantity(distributed_header.rows), - formatReadableSizeWithBinarySuffix(distributed_header.bytes)); - - RemoteInserter remote{*connection, timeouts, - distributed_header.insert_query, - distributed_header.insert_settings, - distributed_header.client_info}; - bool compression_expected = connection->getCompression() == Protocol::Compression::Enable; - writeRemoteConvert(distributed_header, remote, compression_expected, in, log); - remote.onFinish(); - } - catch (Exception & e) - { - if (thread_trace_context) - thread_trace_context->root_span.addAttribute(std::current_exception()); - - e.addMessage(fmt::format("While sending {}", file_path)); - maybeMarkAsBroken(file_path, e); - throw; - } - catch (...) - { - if (thread_trace_context) - thread_trace_context->root_span.addAttribute(std::current_exception()); - - throw; - } - - auto dir_sync_guard = getDirectorySyncGuard(dir_fsync, disk, relative_path); - markAsSend(file_path); - LOG_TRACE(log, "Finished processing `{}` (took {} ms)", file_path, watch.elapsedMilliseconds()); -} - -struct StorageDistributedDirectoryMonitor::BatchHeader -{ - Settings settings; - String query; - ClientInfo client_info; - Block header; - - BatchHeader(Settings settings_, String query_, ClientInfo client_info_, Block header_) - : settings(std::move(settings_)) - , query(std::move(query_)) - , client_info(std::move(client_info_)) - , header(std::move(header_)) - { - } - - bool operator==(const BatchHeader & other) const - { - return std::tie(settings, query, client_info.query_kind) == - std::tie(other.settings, other.query, other.client_info.query_kind) && - blocksHaveEqualStructure(header, other.header); - } - - struct Hash - { - size_t operator()(const BatchHeader & batch_header) const - { - SipHash hash_state; - hash_state.update(batch_header.query.data(), batch_header.query.size()); - batch_header.header.updateHash(hash_state); - return hash_state.get64(); - } - }; -}; - -struct StorageDistributedDirectoryMonitor::Batch -{ - /// File indexes for this batch. - std::vector file_indices; - size_t total_rows = 0; - size_t total_bytes = 0; - bool recovered = false; - - StorageDistributedDirectoryMonitor & parent; - /// Information about all available indexes (not only for the current batch). - const std::map & file_index_to_path; - - bool split_batch_on_failure = true; - bool fsync = false; - bool dir_fsync = false; - - Batch( - StorageDistributedDirectoryMonitor & parent_, - const std::map & file_index_to_path_) - : parent(parent_) - , file_index_to_path(file_index_to_path_) - , split_batch_on_failure(parent.split_batch_on_failure) - , fsync(parent.storage.getDistributedSettingsRef().fsync_after_insert) - , dir_fsync(parent.dir_fsync) - {} - - bool isEnoughSize() const - { - return (!parent.min_batched_block_size_rows && !parent.min_batched_block_size_bytes) - || (parent.min_batched_block_size_rows && total_rows >= parent.min_batched_block_size_rows) - || (parent.min_batched_block_size_bytes && total_bytes >= parent.min_batched_block_size_bytes); - } - - void send() - { - if (file_indices.empty()) - return; - - CurrentMetrics::Increment metric_increment{CurrentMetrics::DistributedSend}; - - Stopwatch watch; - - if (!recovered) - { - /// For deduplication in Replicated tables to work, in case of error - /// we must try to re-send exactly the same batches. - /// So we save contents of the current batch into the current_batch_file_path file - /// and truncate it afterwards if all went well. - - /// Temporary file is required for atomicity. - String tmp_file{parent.current_batch_file_path + ".tmp"}; - - auto dir_sync_guard = getDirectorySyncGuard(dir_fsync, parent.disk, parent.relative_path); - if (fs::exists(tmp_file)) - LOG_ERROR(parent.log, "Temporary file {} exists. Unclean shutdown?", backQuote(tmp_file)); - - { - WriteBufferFromFile out{tmp_file, O_WRONLY | O_TRUNC | O_CREAT}; - writeText(out); - - out.finalize(); - if (fsync) - out.sync(); - } - - fs::rename(tmp_file, parent.current_batch_file_path); - } - - bool batch_broken = false; - bool batch_marked_as_broken = false; - try - { - try - { - sendBatch(); - } - catch (const Exception & e) - { - if (split_batch_on_failure && file_indices.size() > 1 && isSplittableErrorCode(e.code(), e.isRemoteException())) - { - tryLogCurrentException(parent.log, "Trying to split batch due to"); - sendSeparateFiles(); - } - else - throw; - } - } - catch (Exception & e) - { - if (isFileBrokenErrorCode(e.code(), e.isRemoteException())) - { - tryLogCurrentException(parent.log, "Failed to send batch due to"); - batch_broken = true; - if (!e.isRemoteException() && e.code() == ErrorCodes::DISTRIBUTED_BROKEN_BATCH_FILES) - batch_marked_as_broken = true; - } - else - { - std::vector files; - for (auto file_index_info : file_indices | boost::adaptors::indexed()) - { - if (file_index_info.index() > 8) - { - files.push_back("..."); - break; - } - - auto file_index = file_index_info.value(); - auto file_path = file_index_to_path.find(file_index); - if (file_path != file_index_to_path.end()) - files.push_back(file_path->second); - else - files.push_back(fmt::format("#{}.bin (deleted)", file_index)); - } - e.addMessage(fmt::format("While sending batch, size: {}, files: {}", file_indices.size(), fmt::join(files, "\n"))); - - throw; - } - } - - if (!batch_broken) - { - LOG_TRACE(parent.log, "Sent a batch of {} files (took {} ms).", file_indices.size(), watch.elapsedMilliseconds()); - - auto dir_sync_guard = getDirectorySyncGuard(dir_fsync, parent.disk, parent.relative_path); - for (UInt64 file_index : file_indices) - parent.markAsSend(file_index_to_path.at(file_index)); - } - else if (!batch_marked_as_broken) - { - LOG_ERROR(parent.log, "Marking a batch of {} files as broken.", file_indices.size()); - - for (UInt64 file_idx : file_indices) - { - auto file_path = file_index_to_path.find(file_idx); - if (file_path != file_index_to_path.end()) - parent.markAsBroken(file_path->second); - } - } - - file_indices.clear(); - total_rows = 0; - total_bytes = 0; - recovered = false; - - fs::resize_file(parent.current_batch_file_path, 0); - } - - void writeText(WriteBuffer & out) - { - for (UInt64 file_idx : file_indices) - out << file_idx << '\n'; - } - - void readText(ReadBuffer & in) - { - while (!in.eof()) - { - UInt64 idx; - in >> idx >> "\n"; - file_indices.push_back(idx); - } - recovered = true; - } - -private: - void sendBatch() - { - std::unique_ptr remote; - bool compression_expected = false; - - IConnectionPool::Entry connection; - - for (UInt64 file_idx : file_indices) - { - auto file_path = file_index_to_path.find(file_idx); - if (file_path == file_index_to_path.end()) - throw Exception(ErrorCodes::DISTRIBUTED_BROKEN_BATCH_INFO, - "Failed to send batch: file with index {} is absent", file_idx); - - ReadBufferFromFile in(file_path->second); - const auto & distributed_header = readDistributedHeader(in, parent.log); - - OpenTelemetry::TracingContextHolder thread_trace_context(__PRETTY_FUNCTION__, - distributed_header.client_info.client_trace_context, - parent.storage.getContext()->getOpenTelemetrySpanLog()); - - if (!remote) - { - auto timeouts = ConnectionTimeouts::getTCPTimeoutsWithFailover(distributed_header.insert_settings); - connection = parent.pool->get(timeouts); - compression_expected = connection->getCompression() == Protocol::Compression::Enable; - - LOG_DEBUG(parent.log, "Sending a batch of {} files to {} ({} rows, {} bytes).", - file_indices.size(), - connection->getDescription(), - formatReadableQuantity(total_rows), - formatReadableSizeWithBinarySuffix(total_bytes)); - - remote = std::make_unique(*connection, timeouts, - distributed_header.insert_query, - distributed_header.insert_settings, - distributed_header.client_info); - } - writeRemoteConvert(distributed_header, *remote, compression_expected, in, parent.log); - } - - if (remote) - remote->onFinish(); - } - - void sendSeparateFiles() - { - size_t broken_files = 0; - - for (UInt64 file_idx : file_indices) - { - auto file_path = file_index_to_path.find(file_idx); - if (file_path == file_index_to_path.end()) - { - LOG_ERROR(parent.log, "Failed to send one file from batch: file with index {} is absent", file_idx); - ++broken_files; - continue; - } - - try - { - ReadBufferFromFile in(file_path->second); - const auto & distributed_header = readDistributedHeader(in, parent.log); - - // this function is called in a separated thread, so we set up the trace context from the file - OpenTelemetry::TracingContextHolder thread_trace_context(__PRETTY_FUNCTION__, - distributed_header.client_info.client_trace_context, - parent.storage.getContext()->getOpenTelemetrySpanLog()); - - auto timeouts = ConnectionTimeouts::getTCPTimeoutsWithFailover(distributed_header.insert_settings); - auto connection = parent.pool->get(timeouts); - bool compression_expected = connection->getCompression() == Protocol::Compression::Enable; - - RemoteInserter remote(*connection, timeouts, - distributed_header.insert_query, - distributed_header.insert_settings, - distributed_header.client_info); - - writeRemoteConvert(distributed_header, remote, compression_expected, in, parent.log); - remote.onFinish(); - } - catch (Exception & e) - { - e.addMessage(fmt::format("While sending {}", file_path->second)); - parent.maybeMarkAsBroken(file_path->second, e); - ++broken_files; - } - } - - if (broken_files) - throw Exception(ErrorCodes::DISTRIBUTED_BROKEN_BATCH_FILES, - "Failed to send {} files", broken_files); - } -}; - -class DirectoryMonitorSource : public ISource -{ -public: - - struct Data - { - std::unique_ptr in; - std::unique_ptr decompressing_in; - std::unique_ptr block_in; - - Poco::Logger * log = nullptr; - - Block first_block; - - explicit Data(const String & file_name) - { - in = std::make_unique(file_name); - decompressing_in = std::make_unique(*in); - log = &Poco::Logger::get("DirectoryMonitorSource"); - - auto distributed_header = readDistributedHeader(*in, log); - block_in = std::make_unique(*decompressing_in, distributed_header.revision); - - first_block = block_in->read(); - } - - Data(Data &&) = default; - }; - - explicit DirectoryMonitorSource(const String & file_name) - : DirectoryMonitorSource(Data(file_name)) - { - } - - explicit DirectoryMonitorSource(Data data_) - : ISource(data_.first_block.cloneEmpty()) - , data(std::move(data_)) - { - } - - String getName() const override { return "DirectoryMonitorSource"; } - -protected: - Chunk generate() override - { - if (data.first_block) - { - size_t num_rows = data.first_block.rows(); - Chunk res(data.first_block.getColumns(), num_rows); - data.first_block.clear(); - return res; - } - - auto block = data.block_in->read(); - if (!block) - return {}; - - size_t num_rows = block.rows(); - return Chunk(block.getColumns(), num_rows); - } - -private: - Data data; -}; - -std::shared_ptr StorageDistributedDirectoryMonitor::createSourceFromFile(const String & file_name) -{ - return std::make_shared(file_name); -} - -bool StorageDistributedDirectoryMonitor::addAndSchedule(size_t file_size, size_t ms) -{ - if (quit) - return false; - - { - std::lock_guard status_lock(status_mutex); - metric_pending_files.add(); - status.bytes_count += file_size; - ++status.files_count; - } - - return task_handle->scheduleAfter(ms, false); -} - -StorageDistributedDirectoryMonitor::Status StorageDistributedDirectoryMonitor::getStatus() -{ - std::lock_guard status_lock(status_mutex); - Status current_status{status, path, monitor_blocker.isCancelled()}; - return current_status; -} - -void StorageDistributedDirectoryMonitor::processFilesWithBatching(const std::map & files) -{ - std::unordered_set file_indices_to_skip; - - if (fs::exists(current_batch_file_path)) - { - /// Possibly, we failed to send a batch on the previous iteration. Try to send exactly the same batch. - Batch batch(*this, files); - ReadBufferFromFile in{current_batch_file_path}; - batch.readText(in); - file_indices_to_skip.insert(batch.file_indices.begin(), batch.file_indices.end()); - batch.send(); - } - - std::unordered_map header_to_batch; - - for (const auto & file : files) - { - if (quit) - return; - - UInt64 file_idx = file.first; - const String & file_path = file.second; - - if (file_indices_to_skip.contains(file_idx)) - continue; - - size_t total_rows = 0; - size_t total_bytes = 0; - Block header; - DistributedHeader distributed_header; - try - { - /// Determine metadata of the current file and check if it is not broken. - ReadBufferFromFile in{file_path}; - distributed_header = readDistributedHeader(in, log); - - if (distributed_header.rows) - { - total_rows += distributed_header.rows; - total_bytes += distributed_header.bytes; - } - - if (distributed_header.block_header) - header = distributed_header.block_header; - - if (!total_rows || !header) - { - LOG_DEBUG(log, "Processing batch {} with old format (no header/rows)", in.getFileName()); - - CompressedReadBuffer decompressing_in(in); - NativeReader block_in(decompressing_in, distributed_header.revision); - - while (Block block = block_in.read()) - { - total_rows += block.rows(); - total_bytes += block.bytes(); - - if (!header) - header = block.cloneEmpty(); - } - } - } - catch (const Exception & e) - { - if (maybeMarkAsBroken(file_path, e)) - { - tryLogCurrentException(log, "File is marked broken due to"); - continue; - } - else - throw; - } - - BatchHeader batch_header( - std::move(distributed_header.insert_settings), - std::move(distributed_header.insert_query), - std::move(distributed_header.client_info), - std::move(header) - ); - Batch & batch = header_to_batch.try_emplace(batch_header, *this, files).first->second; - - batch.file_indices.push_back(file_idx); - batch.total_rows += total_rows; - batch.total_bytes += total_bytes; - - if (batch.isEnoughSize()) - { - batch.send(); - } - } - - for (auto & kv : header_to_batch) - { - Batch & batch = kv.second; - batch.send(); - } - - { - auto dir_sync_guard = getDirectorySyncGuard(dir_fsync, disk, relative_path); - - /// current_batch.txt will not exist if there was no send - /// (this is the case when all batches that was pending has been marked as pending) - if (fs::exists(current_batch_file_path)) - fs::remove(current_batch_file_path); - } -} - -void StorageDistributedDirectoryMonitor::markAsBroken(const std::string & file_path) -{ - const auto last_path_separator_pos = file_path.rfind('/'); - const auto & base_path = file_path.substr(0, last_path_separator_pos + 1); - const auto & file_name = file_path.substr(last_path_separator_pos + 1); - const String & broken_path = fs::path(base_path) / "broken/"; - const String & broken_file_path = fs::path(broken_path) / file_name; - - fs::create_directory(broken_path); - - auto dir_sync_guard = getDirectorySyncGuard(dir_fsync, disk, relative_path); - auto broken_dir_sync_guard = getDirectorySyncGuard(dir_fsync, disk, fs::path(relative_path) / "broken/"); - - { - std::lock_guard status_lock(status_mutex); - - size_t file_size = fs::file_size(file_path); - - --status.files_count; - status.bytes_count -= file_size; - - ++status.broken_files_count; - status.broken_bytes_count += file_size; - - metric_broken_files.add(); - } - - fs::rename(file_path, broken_file_path); - LOG_ERROR(log, "Renamed `{}` to `{}`", file_path, broken_file_path); -} - -void StorageDistributedDirectoryMonitor::markAsSend(const std::string & file_path) -{ - size_t file_size = fs::file_size(file_path); - - { - std::lock_guard status_lock(status_mutex); - metric_pending_files.sub(); - --status.files_count; - status.bytes_count -= file_size; - } - - fs::remove(file_path); -} - -bool StorageDistributedDirectoryMonitor::maybeMarkAsBroken(const std::string & file_path, const Exception & e) -{ - /// Mark file as broken if necessary. - if (isFileBrokenErrorCode(e.code(), e.isRemoteException())) - { - markAsBroken(file_path); - return true; - } - else - return false; -} - -std::string StorageDistributedDirectoryMonitor::getLoggerName() const -{ - return storage.getStorageID().getFullTableName() + ".DirectoryMonitor"; -} - -void StorageDistributedDirectoryMonitor::updatePath(const std::string & new_relative_path) -{ - task_handle->deactivate(); - std::lock_guard lock{mutex}; - - { - std::lock_guard status_lock(status_mutex); - relative_path = new_relative_path; - path = fs::path(disk->getPath()) / relative_path / ""; - } - current_batch_file_path = path + "current_batch.txt"; - - task_handle->activateAndSchedule(); -} - -} diff --git a/src/Storages/Distributed/DistributedAsyncInsertBatch.cpp b/src/Storages/Distributed/DistributedAsyncInsertBatch.cpp new file mode 100644 index 00000000000..bf410eed6cc --- /dev/null +++ b/src/Storages/Distributed/DistributedAsyncInsertBatch.cpp @@ -0,0 +1,304 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace +{ + +namespace fs = std::filesystem; + +} + +namespace CurrentMetrics +{ + extern const Metric DistributedSend; +} + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int MEMORY_LIMIT_EXCEEDED; + extern const int DISTRIBUTED_BROKEN_BATCH_INFO; + extern const int DISTRIBUTED_BROKEN_BATCH_FILES; + extern const int TOO_MANY_PARTS; + extern const int TOO_MANY_BYTES; + extern const int TOO_MANY_ROWS_OR_BYTES; + extern const int TOO_MANY_PARTITIONS; + extern const int DISTRIBUTED_TOO_MANY_PENDING_BYTES; + extern const int ARGUMENT_OUT_OF_BOUND; +} + +/// Can the batch be split and send files from batch one-by-one instead? +bool isSplittableErrorCode(int code, bool remote) +{ + return code == ErrorCodes::MEMORY_LIMIT_EXCEEDED + /// FunctionRange::max_elements and similar + || code == ErrorCodes::ARGUMENT_OUT_OF_BOUND + || code == ErrorCodes::TOO_MANY_PARTS + || code == ErrorCodes::TOO_MANY_BYTES + || code == ErrorCodes::TOO_MANY_ROWS_OR_BYTES + || code == ErrorCodes::TOO_MANY_PARTITIONS + || code == ErrorCodes::DISTRIBUTED_TOO_MANY_PENDING_BYTES + || code == ErrorCodes::DISTRIBUTED_BROKEN_BATCH_INFO + || isDistributedSendBroken(code, remote) + ; +} + +DistributedAsyncInsertBatch::DistributedAsyncInsertBatch(DistributedAsyncInsertDirectoryQueue & parent_) + : parent(parent_) + , split_batch_on_failure(parent.split_batch_on_failure) + , fsync(parent.storage.getDistributedSettingsRef().fsync_after_insert) + , dir_fsync(parent.dir_fsync) +{} + +bool DistributedAsyncInsertBatch::isEnoughSize() const +{ + return (!parent.min_batched_block_size_rows && !parent.min_batched_block_size_bytes) + || (parent.min_batched_block_size_rows && total_rows >= parent.min_batched_block_size_rows) + || (parent.min_batched_block_size_bytes && total_bytes >= parent.min_batched_block_size_bytes); +} + +void DistributedAsyncInsertBatch::send() +{ + if (files.empty()) + return; + + CurrentMetrics::Increment metric_increment{CurrentMetrics::DistributedSend}; + + Stopwatch watch; + + if (!recovered) + { + /// For deduplication in Replicated tables to work, in case of error + /// we must try to re-send exactly the same batches. + /// So we save contents of the current batch into the current_batch_file_path file + /// and truncate it afterwards if all went well. + serialize(); + } + + bool batch_broken = false; + bool batch_marked_as_broken = false; + try + { + try + { + sendBatch(); + } + catch (const Exception & e) + { + if (split_batch_on_failure && files.size() > 1 && isSplittableErrorCode(e.code(), e.isRemoteException())) + { + tryLogCurrentException(parent.log, "Trying to split batch due to"); + sendSeparateFiles(); + } + else + throw; + } + } + catch (Exception & e) + { + if (isDistributedSendBroken(e.code(), e.isRemoteException())) + { + tryLogCurrentException(parent.log, "Failed to send batch due to"); + batch_broken = true; + if (!e.isRemoteException() && e.code() == ErrorCodes::DISTRIBUTED_BROKEN_BATCH_FILES) + batch_marked_as_broken = true; + } + else + { + e.addMessage(fmt::format("While sending a batch of {} files, files: {}", files.size(), fmt::join(files, "\n"))); + throw; + } + } + + if (!batch_broken) + { + LOG_TRACE(parent.log, "Sent a batch of {} files (took {} ms).", files.size(), watch.elapsedMilliseconds()); + + auto dir_sync_guard = parent.getDirectorySyncGuard(parent.relative_path); + for (const auto & file : files) + parent.markAsSend(file); + } + else if (!batch_marked_as_broken) + { + LOG_ERROR(parent.log, "Marking a batch of {} files as broken, files: {}", files.size(), fmt::join(files, "\n")); + + for (const auto & file : files) + parent.markAsBroken(file); + } + + files.clear(); + total_rows = 0; + total_bytes = 0; + recovered = false; + + fs::resize_file(parent.current_batch_file_path, 0); +} + +void DistributedAsyncInsertBatch::serialize() +{ + /// Temporary file is required for atomicity. + String tmp_file{parent.current_batch_file_path + ".tmp"}; + + auto dir_sync_guard = parent.getDirectorySyncGuard(parent.relative_path); + if (fs::exists(tmp_file)) + LOG_ERROR(parent.log, "Temporary file {} exists. Unclean shutdown?", backQuote(tmp_file)); + + { + WriteBufferFromFile out{tmp_file, O_WRONLY | O_TRUNC | O_CREAT}; + writeText(out); + + out.finalize(); + if (fsync) + out.sync(); + } + + fs::rename(tmp_file, parent.current_batch_file_path); +} + +void DistributedAsyncInsertBatch::deserialize() +{ + ReadBufferFromFile in{parent.current_batch_file_path}; + readText(in); +} + +void DistributedAsyncInsertBatch::writeText(WriteBuffer & out) +{ + for (const auto & file : files) + { + UInt64 file_index = parse(fs::path(file).stem()); + out << file_index << '\n'; + } +} + +void DistributedAsyncInsertBatch::readText(ReadBuffer & in) +{ + while (!in.eof()) + { + UInt64 idx; + in >> idx >> "\n"; + files.push_back(fs::absolute(fmt::format("{}/{}.bin", parent.path, idx)).string()); + } + + recovered = true; +} + +void DistributedAsyncInsertBatch::sendBatch() +{ + std::unique_ptr remote; + bool compression_expected = false; + + IConnectionPool::Entry connection; + + /// Since the batch is sent as a whole (in case of failure, the whole batch + /// will be repeated), we need to mark the whole batch as failed in case of + /// error). + std::vector tracing_contexts; + UInt64 batch_start_time = clock_gettime_ns(); + + try + { + for (const auto & file : files) + { + ReadBufferFromFile in(file); + const auto & distributed_header = DistributedAsyncInsertHeader::read(in, parent.log); + + tracing_contexts.emplace_back(distributed_header.createTracingContextHolder( + __PRETTY_FUNCTION__, + parent.storage.getContext()->getOpenTelemetrySpanLog())); + tracing_contexts.back()->root_span.addAttribute("clickhouse.distributed_batch_start_time", batch_start_time); + + if (!remote) + { + auto timeouts = ConnectionTimeouts::getTCPTimeoutsWithFailover(distributed_header.insert_settings); + connection = parent.pool->get(timeouts); + compression_expected = connection->getCompression() == Protocol::Compression::Enable; + + LOG_DEBUG(parent.log, "Sending a batch of {} files to {} ({} rows, {} bytes).", + files.size(), + connection->getDescription(), + formatReadableQuantity(total_rows), + formatReadableSizeWithBinarySuffix(total_bytes)); + + remote = std::make_unique(*connection, timeouts, + distributed_header.insert_query, + distributed_header.insert_settings, + distributed_header.client_info); + } + writeRemoteConvert(distributed_header, *remote, compression_expected, in, parent.log); + } + + if (remote) + remote->onFinish(); + } + catch (...) + { + try + { + for (auto & tracing_context : tracing_contexts) + tracing_context->root_span.addAttribute(std::current_exception()); + } + catch (...) + { + tryLogCurrentException(parent.log, "Cannot append exception to tracing context"); + } + throw; + } +} + +void DistributedAsyncInsertBatch::sendSeparateFiles() +{ + size_t broken_files = 0; + + for (const auto & file : files) + { + OpenTelemetry::TracingContextHolderPtr trace_context; + + try + { + ReadBufferFromFile in(file); + const auto & distributed_header = DistributedAsyncInsertHeader::read(in, parent.log); + + // This function is called in a separated thread, so we set up the trace context from the file + trace_context = distributed_header.createTracingContextHolder( + __PRETTY_FUNCTION__, + parent.storage.getContext()->getOpenTelemetrySpanLog()); + + auto timeouts = ConnectionTimeouts::getTCPTimeoutsWithFailover(distributed_header.insert_settings); + auto connection = parent.pool->get(timeouts); + bool compression_expected = connection->getCompression() == Protocol::Compression::Enable; + + RemoteInserter remote(*connection, timeouts, + distributed_header.insert_query, + distributed_header.insert_settings, + distributed_header.client_info); + + writeRemoteConvert(distributed_header, remote, compression_expected, in, parent.log); + remote.onFinish(); + } + catch (Exception & e) + { + trace_context->root_span.addAttribute(std::current_exception()); + + if (isDistributedSendBroken(e.code(), e.isRemoteException())) + { + parent.markAsBroken(file); + ++broken_files; + } + } + } + + if (broken_files) + throw Exception(ErrorCodes::DISTRIBUTED_BROKEN_BATCH_FILES, + "Failed to send {} files", broken_files); +} + +} diff --git a/src/Storages/Distributed/DistributedAsyncInsertBatch.h b/src/Storages/Distributed/DistributedAsyncInsertBatch.h new file mode 100644 index 00000000000..867a0de89fa --- /dev/null +++ b/src/Storages/Distributed/DistributedAsyncInsertBatch.h @@ -0,0 +1,44 @@ +#pragma once + +#include +#include + +namespace DB +{ + +class DistributedAsyncInsertDirectoryQueue; +class WriteBuffer; +class ReadBuffer; + +class DistributedAsyncInsertBatch +{ +public: + explicit DistributedAsyncInsertBatch(DistributedAsyncInsertDirectoryQueue & parent_); + + bool isEnoughSize() const; + void send(); + + void serialize(); + void deserialize(); + + size_t total_rows = 0; + size_t total_bytes = 0; + std::vector files; + +private: + void writeText(WriteBuffer & out); + void readText(ReadBuffer & in); + void sendBatch(); + void sendSeparateFiles(); + + DistributedAsyncInsertDirectoryQueue & parent; + + /// Does the batch had been created from the files in current_batch.txt? + bool recovered = false; + + bool split_batch_on_failure = true; + bool fsync = false; + bool dir_fsync = false; +}; + +} diff --git a/src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.cpp b/src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.cpp new file mode 100644 index 00000000000..9a9a6651bc4 --- /dev/null +++ b/src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.cpp @@ -0,0 +1,729 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace CurrentMetrics +{ + extern const Metric DistributedSend; + extern const Metric DistributedFilesToInsert; + extern const Metric BrokenDistributedFilesToInsert; +} + +namespace fs = std::filesystem; + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int INCORRECT_FILE_NAME; + extern const int LOGICAL_ERROR; +} + + +namespace +{ + +template +ConnectionPoolPtrs createPoolsForAddresses(const std::string & name, PoolFactory && factory, const Cluster::ShardsInfo & shards_info, Poco::Logger * log) +{ + ConnectionPoolPtrs pools; + + auto make_connection = [&](const Cluster::Address & address) + { + try + { + pools.emplace_back(factory(address)); + } + catch (const Exception & e) + { + if (e.code() == ErrorCodes::INCORRECT_FILE_NAME) + { + tryLogCurrentException(log); + return; + } + throw; + } + }; + + for (auto it = boost::make_split_iterator(name, boost::first_finder(",")); it != decltype(it){}; ++it) + { + const std::string & dirname = boost::copy_range(*it); + Cluster::Address address = Cluster::Address::fromFullString(dirname); + if (address.shard_index && dirname.ends_with("_all_replicas")) + { + if (address.shard_index > shards_info.size()) + { + LOG_ERROR(log, "No shard with shard_index={} ({})", address.shard_index, name); + continue; + } + + const auto & shard_info = shards_info[address.shard_index - 1]; + size_t replicas = shard_info.per_replica_pools.size(); + + for (size_t replica_index = 1; replica_index <= replicas; ++replica_index) + { + address.replica_index = static_cast(replica_index); + make_connection(address); + } + } + else + make_connection(address); + } + + return pools; +} + +uint64_t doubleToUInt64(double d) +{ + if (d >= static_cast(std::numeric_limits::max())) + return std::numeric_limits::max(); + return static_cast(d); +} + +} + + +DistributedAsyncInsertDirectoryQueue::DistributedAsyncInsertDirectoryQueue( + StorageDistributed & storage_, + const DiskPtr & disk_, + const std::string & relative_path_, + ConnectionPoolPtr pool_, + ActionBlocker & monitor_blocker_, + BackgroundSchedulePool & bg_pool) + : storage(storage_) + , pool(std::move(pool_)) + , disk(disk_) + , relative_path(relative_path_) + , path(fs::path(disk->getPath()) / relative_path / "") + , broken_relative_path(fs::path(relative_path) / "broken") + , broken_path(fs::path(path) / "broken" / "") + , should_batch_inserts(storage.getDistributedSettingsRef().monitor_batch_inserts) + , split_batch_on_failure(storage.getDistributedSettingsRef().monitor_split_batch_on_failure) + , dir_fsync(storage.getDistributedSettingsRef().fsync_directories) + , min_batched_block_size_rows(storage.getContext()->getSettingsRef().min_insert_block_size_rows) + , min_batched_block_size_bytes(storage.getContext()->getSettingsRef().min_insert_block_size_bytes) + , current_batch_file_path(path + "current_batch.txt") + , pending_files(std::numeric_limits::max()) + , default_sleep_time(storage.getDistributedSettingsRef().monitor_sleep_time_ms.totalMilliseconds()) + , sleep_time(default_sleep_time) + , max_sleep_time(storage.getDistributedSettingsRef().monitor_max_sleep_time_ms.totalMilliseconds()) + , log(&Poco::Logger::get(getLoggerName())) + , monitor_blocker(monitor_blocker_) + , metric_pending_files(CurrentMetrics::DistributedFilesToInsert, 0) + , metric_broken_files(CurrentMetrics::BrokenDistributedFilesToInsert, 0) +{ + fs::create_directory(broken_path); + + initializeFilesFromDisk(); + + task_handle = bg_pool.createTask(getLoggerName() + "/Bg", [this]{ run(); }); + task_handle->activateAndSchedule(); +} + + +DistributedAsyncInsertDirectoryQueue::~DistributedAsyncInsertDirectoryQueue() +{ + if (!pending_files.isFinished()) + { + pending_files.clearAndFinish(); + task_handle->deactivate(); + } +} + +void DistributedAsyncInsertDirectoryQueue::flushAllData() +{ + if (pending_files.isFinished()) + return; + + std::lock_guard lock{mutex}; + if (!hasPendingFiles()) + return; + processFiles(); +} + +void DistributedAsyncInsertDirectoryQueue::shutdownAndDropAllData() +{ + if (!pending_files.isFinished()) + { + pending_files.clearAndFinish(); + task_handle->deactivate(); + } + + auto dir_sync_guard = getDirectorySyncGuard(relative_path); + fs::remove_all(path); +} + + +void DistributedAsyncInsertDirectoryQueue::run() +{ + constexpr const std::chrono::minutes decrease_error_count_period{5}; + + std::lock_guard lock{mutex}; + + bool do_sleep = false; + while (!pending_files.isFinished()) + { + do_sleep = true; + + if (!hasPendingFiles()) + break; + + if (!monitor_blocker.isCancelled()) + { + try + { + processFiles(); + /// No errors while processing existing files. + /// Let's see maybe there are more files to process. + do_sleep = false; + } + catch (...) + { + tryLogCurrentException(getLoggerName().data()); + + UInt64 q = doubleToUInt64(std::exp2(status.error_count)); + std::chrono::milliseconds new_sleep_time(default_sleep_time.count() * q); + if (new_sleep_time.count() < 0) + sleep_time = max_sleep_time; + else + sleep_time = std::min(new_sleep_time, max_sleep_time); + + do_sleep = true; + } + } + else + LOG_TEST(log, "Skipping send data over distributed table."); + + const auto now = std::chrono::system_clock::now(); + if (now - last_decrease_time > decrease_error_count_period) + { + std::lock_guard status_lock(status_mutex); + + status.error_count /= 2; + last_decrease_time = now; + } + + if (do_sleep) + break; + } + + if (!pending_files.isFinished() && do_sleep) + task_handle->scheduleAfter(sleep_time.count()); +} + + +ConnectionPoolPtr DistributedAsyncInsertDirectoryQueue::createPool(const std::string & name, const StorageDistributed & storage) +{ + const auto pool_factory = [&storage, &name] (const Cluster::Address & address) -> ConnectionPoolPtr + { + const auto & cluster = storage.getCluster(); + const auto & shards_info = cluster->getShardsInfo(); + const auto & shards_addresses = cluster->getShardsAddresses(); + + /// Check new format shard{shard_index}_replica{replica_index} + /// (shard_index and replica_index starts from 1). + if (address.shard_index != 0) + { + if (!address.replica_index) + throw Exception(ErrorCodes::INCORRECT_FILE_NAME, + "Wrong replica_index={} ({})", address.replica_index, name); + + if (address.shard_index > shards_info.size()) + throw Exception(ErrorCodes::INCORRECT_FILE_NAME, + "No shard with shard_index={} ({})", address.shard_index, name); + + const auto & shard_info = shards_info[address.shard_index - 1]; + if (address.replica_index > shard_info.per_replica_pools.size()) + throw Exception(ErrorCodes::INCORRECT_FILE_NAME, + "No shard with replica_index={} ({})", address.replica_index, name); + + return shard_info.per_replica_pools[address.replica_index - 1]; + } + + /// Existing connections pool have a higher priority. + for (size_t shard_index = 0; shard_index < shards_info.size(); ++shard_index) + { + const Cluster::Addresses & replicas_addresses = shards_addresses[shard_index]; + + for (size_t replica_index = 0; replica_index < replicas_addresses.size(); ++replica_index) + { + const Cluster::Address & replica_address = replicas_addresses[replica_index]; + + if (address.user == replica_address.user && + address.password == replica_address.password && + address.host_name == replica_address.host_name && + address.port == replica_address.port && + address.default_database == replica_address.default_database && + address.secure == replica_address.secure) + { + return shards_info[shard_index].per_replica_pools[replica_index]; + } + } + } + + return std::make_shared( + 1, /* max_connections */ + address.host_name, + address.port, + address.default_database, + address.user, + address.password, + address.quota_key, + address.cluster, + address.cluster_secret, + storage.getName() + '_' + address.user, /* client */ + Protocol::Compression::Enable, + address.secure); + }; + + auto pools = createPoolsForAddresses(name, pool_factory, storage.getCluster()->getShardsInfo(), storage.log); + + const auto settings = storage.getContext()->getSettings(); + return pools.size() == 1 ? pools.front() : std::make_shared(pools, + settings.load_balancing, + settings.distributed_replica_error_half_life.totalSeconds(), + settings.distributed_replica_error_cap); +} + +bool DistributedAsyncInsertDirectoryQueue::hasPendingFiles() const +{ + return fs::exists(current_batch_file_path) || !current_file.empty() || !pending_files.empty(); +} + +void DistributedAsyncInsertDirectoryQueue::addFile(const std::string & file_path) +{ + if (!pending_files.push(fs::absolute(file_path).string())) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot schedule a file '{}'", file_path); +} + +void DistributedAsyncInsertDirectoryQueue::initializeFilesFromDisk() +{ + /// NOTE: This method does not requires to hold status_mutex (because this + /// object is not in the list that the caller may iterate over), hence, no + /// TSA annotations in the header file. + + fs::directory_iterator end; + + /// Initialize pending files + { + size_t bytes_count = 0; + + for (fs::directory_iterator it{path}; it != end; ++it) + { + const auto & file_path = it->path(); + const auto & base_name = file_path.stem().string(); + if (!it->is_directory() && startsWith(fs::path(file_path).extension(), ".bin") && parse(base_name)) + { + const std::string & file_path_str = file_path.string(); + addFile(file_path_str); + bytes_count += fs::file_size(file_path); + } + else if (base_name != "tmp" && base_name != "broken") + { + /// It is OK to log current_batch.txt here too (useful for debugging). + LOG_WARNING(log, "Unexpected file {} in {}", file_path.string(), path); + } + } + + LOG_TRACE(log, "Files set to {}", pending_files.size()); + LOG_TRACE(log, "Bytes set to {}", bytes_count); + + metric_pending_files.changeTo(pending_files.size()); + status.files_count = pending_files.size(); + status.bytes_count = bytes_count; + } + + /// Initialize broken files + { + size_t broken_bytes_count = 0; + size_t broken_files = 0; + + for (fs::directory_iterator it{broken_path}; it != end; ++it) + { + const auto & file_path = it->path(); + if (!it->is_directory() && startsWith(fs::path(file_path).extension(), ".bin") && parse(file_path.stem())) + broken_bytes_count += fs::file_size(file_path); + else + LOG_WARNING(log, "Unexpected file {} in {}", file_path.string(), broken_path); + } + + LOG_TRACE(log, "Broken files set to {}", broken_files); + LOG_TRACE(log, "Broken bytes set to {}", broken_bytes_count); + + metric_broken_files.changeTo(broken_files); + status.broken_files_count = broken_files; + status.broken_bytes_count = broken_bytes_count; + } +} +void DistributedAsyncInsertDirectoryQueue::processFiles() +try +{ + if (should_batch_inserts) + processFilesWithBatching(); + else + { + /// Process unprocessed file. + if (!current_file.empty()) + processFile(current_file); + + while (pending_files.tryPop(current_file)) + processFile(current_file); + } + + std::lock_guard status_lock(status_mutex); + status.last_exception = std::exception_ptr{}; +} +catch (...) +{ + std::lock_guard status_lock(status_mutex); + + ++status.error_count; + status.last_exception = std::current_exception(); + status.last_exception_time = std::chrono::system_clock::now(); + + throw; +} + +void DistributedAsyncInsertDirectoryQueue::processFile(const std::string & file_path) +{ + OpenTelemetry::TracingContextHolderPtr thread_trace_context; + + Stopwatch watch; + try + { + CurrentMetrics::Increment metric_increment{CurrentMetrics::DistributedSend}; + + ReadBufferFromFile in(file_path); + const auto & distributed_header = DistributedAsyncInsertHeader::read(in, log); + thread_trace_context = distributed_header.createTracingContextHolder( + __PRETTY_FUNCTION__, + storage.getContext()->getOpenTelemetrySpanLog()); + + auto timeouts = ConnectionTimeouts::getTCPTimeoutsWithFailover(distributed_header.insert_settings); + auto connection = pool->get(timeouts, &distributed_header.insert_settings); + LOG_DEBUG(log, "Sending `{}` to {} ({} rows, {} bytes)", + file_path, + connection->getDescription(), + formatReadableQuantity(distributed_header.rows), + formatReadableSizeWithBinarySuffix(distributed_header.bytes)); + + RemoteInserter remote{*connection, timeouts, + distributed_header.insert_query, + distributed_header.insert_settings, + distributed_header.client_info}; + bool compression_expected = connection->getCompression() == Protocol::Compression::Enable; + writeRemoteConvert(distributed_header, remote, compression_expected, in, log); + remote.onFinish(); + } + catch (Exception & e) + { + if (thread_trace_context) + thread_trace_context->root_span.addAttribute(std::current_exception()); + + e.addMessage(fmt::format("While sending {}", file_path)); + if (isDistributedSendBroken(e.code(), e.isRemoteException())) + { + markAsBroken(file_path); + current_file.clear(); + } + throw; + } + catch (...) + { + if (thread_trace_context) + thread_trace_context->root_span.addAttribute(std::current_exception()); + + throw; + } + + auto dir_sync_guard = getDirectorySyncGuard(relative_path); + markAsSend(file_path); + current_file.clear(); + LOG_TRACE(log, "Finished processing `{}` (took {} ms)", file_path, watch.elapsedMilliseconds()); +} + +struct DistributedAsyncInsertDirectoryQueue::BatchHeader +{ + Settings settings; + String query; + ClientInfo client_info; + Block header; + + BatchHeader(Settings settings_, String query_, ClientInfo client_info_, Block header_) + : settings(std::move(settings_)) + , query(std::move(query_)) + , client_info(std::move(client_info_)) + , header(std::move(header_)) + { + } + + bool operator==(const BatchHeader & other) const + { + return std::tie(settings, query, client_info.query_kind) == + std::tie(other.settings, other.query, other.client_info.query_kind) && + blocksHaveEqualStructure(header, other.header); + } + + struct Hash + { + size_t operator()(const BatchHeader & batch_header) const + { + SipHash hash_state; + hash_state.update(batch_header.query.data(), batch_header.query.size()); + batch_header.header.updateHash(hash_state); + return hash_state.get64(); + } + }; +}; + +bool DistributedAsyncInsertDirectoryQueue::addFileAndSchedule(const std::string & file_path, size_t file_size, size_t ms) +{ + /// NOTE: It is better not to throw in this case, since the file is already + /// on disk (see DistributedSink), and it will be processed next time. + if (pending_files.isFinished()) + { + LOG_DEBUG(log, "File {} had not been scheduled, since the table had been detached", file_path); + return false; + } + + addFile(file_path); + + { + std::lock_guard lock(status_mutex); + metric_pending_files.add(); + status.bytes_count += file_size; + ++status.files_count; + } + + return task_handle->scheduleAfter(ms, false); +} + +DistributedAsyncInsertDirectoryQueue::Status DistributedAsyncInsertDirectoryQueue::getStatus() +{ + std::lock_guard status_lock(status_mutex); + Status current_status{status, path, monitor_blocker.isCancelled()}; + return current_status; +} + +void DistributedAsyncInsertDirectoryQueue::processFilesWithBatching() +{ + /// Possibly, we failed to send a batch on the previous iteration. Try to send exactly the same batch. + if (fs::exists(current_batch_file_path)) + { + LOG_DEBUG(log, "Restoring the batch"); + + DistributedAsyncInsertBatch batch(*this); + batch.deserialize(); + batch.send(); + + auto dir_sync_guard = getDirectorySyncGuard(relative_path); + fs::remove(current_batch_file_path); + } + + std::unordered_map header_to_batch; + + std::string file_path; + + try + { + while (pending_files.tryPop(file_path)) + { + if (!fs::exists(file_path)) + { + LOG_WARNING(log, "File {} does not exists, likely due to current_batch.txt processing", file_path); + continue; + } + + size_t total_rows = 0; + size_t total_bytes = 0; + Block header; + DistributedAsyncInsertHeader distributed_header; + try + { + /// Determine metadata of the current file and check if it is not broken. + ReadBufferFromFile in{file_path}; + distributed_header = DistributedAsyncInsertHeader::read(in, log); + + if (distributed_header.rows) + { + total_rows += distributed_header.rows; + total_bytes += distributed_header.bytes; + } + + if (distributed_header.block_header) + header = distributed_header.block_header; + + if (!total_rows || !header) + { + LOG_DEBUG(log, "Processing batch {} with old format (no header/rows)", in.getFileName()); + + CompressedReadBuffer decompressing_in(in); + NativeReader block_in(decompressing_in, distributed_header.revision); + + while (Block block = block_in.read()) + { + total_rows += block.rows(); + total_bytes += block.bytes(); + + if (!header) + header = block.cloneEmpty(); + } + } + } + catch (const Exception & e) + { + if (isDistributedSendBroken(e.code(), e.isRemoteException())) + { + markAsBroken(file_path); + tryLogCurrentException(log, "File is marked broken due to"); + continue; + } + else + throw; + } + + BatchHeader batch_header( + std::move(distributed_header.insert_settings), + std::move(distributed_header.insert_query), + std::move(distributed_header.client_info), + std::move(header) + ); + DistributedAsyncInsertBatch & batch = header_to_batch.try_emplace(batch_header, *this).first->second; + + batch.files.push_back(file_path); + batch.total_rows += total_rows; + batch.total_bytes += total_bytes; + + if (batch.isEnoughSize()) + { + batch.send(); + } + } + + for (auto & kv : header_to_batch) + { + DistributedAsyncInsertBatch & batch = kv.second; + batch.send(); + } + } + catch (...) + { + /// Revert uncommitted files. + for (const auto & [_, batch] : header_to_batch) + { + for (const auto & file : batch.files) + { + if (!pending_files.pushFront(file)) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot re-schedule a file '{}'", file); + } + } + /// Rethrow exception + throw; + } + + { + auto dir_sync_guard = getDirectorySyncGuard(relative_path); + + /// current_batch.txt will not exist if there was no send + /// (this is the case when all batches that was pending has been marked as pending) + if (fs::exists(current_batch_file_path)) + fs::remove(current_batch_file_path); + } +} + +void DistributedAsyncInsertDirectoryQueue::markAsBroken(const std::string & file_path) +{ + const String & broken_file_path = fs::path(broken_path) / fs::path(file_path).filename(); + + auto dir_sync_guard = getDirectorySyncGuard(relative_path); + auto broken_dir_sync_guard = getDirectorySyncGuard(broken_relative_path); + + { + std::lock_guard status_lock(status_mutex); + + size_t file_size = fs::file_size(file_path); + + --status.files_count; + status.bytes_count -= file_size; + + ++status.broken_files_count; + status.broken_bytes_count += file_size; + + metric_broken_files.add(); + } + + fs::rename(file_path, broken_file_path); + LOG_ERROR(log, "Renamed `{}` to `{}`", file_path, broken_file_path); +} + +void DistributedAsyncInsertDirectoryQueue::markAsSend(const std::string & file_path) +{ + size_t file_size = fs::file_size(file_path); + + { + std::lock_guard status_lock(status_mutex); + metric_pending_files.sub(); + --status.files_count; + status.bytes_count -= file_size; + } + + fs::remove(file_path); +} + +SyncGuardPtr DistributedAsyncInsertDirectoryQueue::getDirectorySyncGuard(const std::string & dir_path) +{ + if (dir_fsync) + return disk->getDirectorySyncGuard(dir_path); + return nullptr; +} + +std::string DistributedAsyncInsertDirectoryQueue::getLoggerName() const +{ + return storage.getStorageID().getFullTableName() + ".DirectoryMonitor." + disk->getName(); +} + +void DistributedAsyncInsertDirectoryQueue::updatePath(const std::string & new_relative_path) +{ + task_handle->deactivate(); + std::lock_guard lock{mutex}; + + { + std::lock_guard status_lock(status_mutex); + relative_path = new_relative_path; + path = fs::path(disk->getPath()) / relative_path / ""; + } + current_batch_file_path = path + "current_batch.txt"; + + task_handle->activateAndSchedule(); +} + +} diff --git a/src/Storages/Distributed/DirectoryMonitor.h b/src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.h similarity index 63% rename from src/Storages/Distributed/DirectoryMonitor.h rename to src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.h index 030d6acf6e2..de8bb813824 100644 --- a/src/Storages/Distributed/DirectoryMonitor.h +++ b/src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.h @@ -1,12 +1,13 @@ #pragma once #include +#include #include - +#include +#include #include #include #include -#include namespace CurrentMetrics { class Increment; } @@ -26,13 +27,28 @@ using ProcessorPtr = std::shared_ptr; class ISource; -/** Details of StorageDistributed. - * This type is not designed for standalone use. - */ -class StorageDistributedDirectoryMonitor +/** Queue for async INSERT Into Distributed engine (insert_distributed_sync=0). + * + * Files are added from two places: + * - from filesystem at startup (StorageDistributed::startup()) + * - on INSERT via DistributedSink + * + * Later, in background, those files will be send to the remote nodes. + * + * The behaviour of this queue can be configured via the following settings: + * - distributed_directory_monitor_batch_inserts + * - distributed_directory_monitor_split_batch_on_failure + * - distributed_directory_monitor_sleep_time_ms + * - distributed_directory_monitor_max_sleep_time_ms + * NOTE: It worth to rename the settings too + * ("directory_monitor" in settings looks too internal). + */ +class DistributedAsyncInsertDirectoryQueue { + friend class DistributedAsyncInsertBatch; + public: - StorageDistributedDirectoryMonitor( + DistributedAsyncInsertDirectoryQueue( StorageDistributed & storage_, const DiskPtr & disk_, const std::string & relative_path_, @@ -40,7 +56,7 @@ public: ActionBlocker & monitor_blocker_, BackgroundSchedulePool & bg_pool); - ~StorageDistributedDirectoryMonitor(); + ~DistributedAsyncInsertDirectoryQueue(); static ConnectionPoolPtr createPool(const std::string & name, const StorageDistributed & storage); @@ -53,7 +69,7 @@ public: static std::shared_ptr createSourceFromFile(const String & file_name); /// For scheduling via DistributedSink. - bool addAndSchedule(size_t file_size, size_t ms); + bool addFileAndSchedule(const std::string & file_path, size_t file_size, size_t ms); struct InternalStatus { @@ -79,14 +95,18 @@ public: private: void run(); - std::map getFiles(); - bool processFiles(const std::map & files); + bool hasPendingFiles() const; + + void addFile(const std::string & file_path); + void initializeFilesFromDisk(); + void processFiles(); void processFile(const std::string & file_path); - void processFilesWithBatching(const std::map & files); + void processFilesWithBatching(); void markAsBroken(const std::string & file_path); void markAsSend(const std::string & file_path); - bool maybeMarkAsBroken(const std::string & file_path, const Exception & e); + + SyncGuardPtr getDirectorySyncGuard(const std::string & path); std::string getLoggerName() const; @@ -96,25 +116,33 @@ private: DiskPtr disk; std::string relative_path; std::string path; + std::string broken_relative_path; + std::string broken_path; const bool should_batch_inserts = false; const bool split_batch_on_failure = true; const bool dir_fsync = false; const size_t min_batched_block_size_rows = 0; const size_t min_batched_block_size_bytes = 0; - String current_batch_file_path; + + /// This is pending data (due to some error) for should_batch_inserts==true + std::string current_batch_file_path; + /// This is pending data (due to some error) for should_batch_inserts==false + std::string current_file; struct BatchHeader; struct Batch; std::mutex status_mutex; + InternalStatus status; + ConcurrentBoundedQueue pending_files; + const std::chrono::milliseconds default_sleep_time; std::chrono::milliseconds sleep_time; const std::chrono::milliseconds max_sleep_time; std::chrono::time_point last_decrease_time {std::chrono::system_clock::now()}; - std::atomic quit {false}; std::mutex mutex; Poco::Logger * log; ActionBlocker & monitor_blocker; diff --git a/src/Storages/Distributed/DistributedAsyncInsertHeader.cpp b/src/Storages/Distributed/DistributedAsyncInsertHeader.cpp new file mode 100644 index 00000000000..018c1d863bb --- /dev/null +++ b/src/Storages/Distributed/DistributedAsyncInsertHeader.cpp @@ -0,0 +1,125 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int CANNOT_READ_ALL_DATA; + extern const int CHECKSUM_DOESNT_MATCH; +} + +DistributedAsyncInsertHeader DistributedAsyncInsertHeader::read(ReadBufferFromFile & in, Poco::Logger * log) +{ + DistributedAsyncInsertHeader distributed_header; + + UInt64 query_size; + readVarUInt(query_size, in); + + if (query_size == DBMS_DISTRIBUTED_SIGNATURE_HEADER) + { + /// Read the header as a string. + String header_data; + readStringBinary(header_data, in); + + /// Check the checksum of the header. + CityHash_v1_0_2::uint128 expected_checksum; + readPODBinary(expected_checksum, in); + CityHash_v1_0_2::uint128 calculated_checksum = + CityHash_v1_0_2::CityHash128(header_data.data(), header_data.size()); + if (expected_checksum != calculated_checksum) + { + throw Exception(ErrorCodes::CHECKSUM_DOESNT_MATCH, + "Checksum of extra info doesn't match: corrupted data. Reference: {}{}. Actual: {}{}.", + getHexUIntLowercase(expected_checksum.first), getHexUIntLowercase(expected_checksum.second), + getHexUIntLowercase(calculated_checksum.first), getHexUIntLowercase(calculated_checksum.second)); + } + + /// Read the parts of the header. + ReadBufferFromString header_buf(header_data); + + readVarUInt(distributed_header.revision, header_buf); + if (DBMS_TCP_PROTOCOL_VERSION < distributed_header.revision) + { + LOG_WARNING(log, "ClickHouse shard version is older than ClickHouse initiator version. It may lack support for new features."); + } + + readStringBinary(distributed_header.insert_query, header_buf); + distributed_header.insert_settings.read(header_buf); + + if (header_buf.hasPendingData()) + distributed_header.client_info.read(header_buf, distributed_header.revision); + + if (header_buf.hasPendingData()) + { + readVarUInt(distributed_header.rows, header_buf); + readVarUInt(distributed_header.bytes, header_buf); + readStringBinary(distributed_header.block_header_string, header_buf); + } + + if (header_buf.hasPendingData()) + { + NativeReader header_block_in(header_buf, distributed_header.revision); + distributed_header.block_header = header_block_in.read(); + if (!distributed_header.block_header) + throw Exception(ErrorCodes::CANNOT_READ_ALL_DATA, + "Cannot read header from the {} batch. Data was written with protocol version {}, current version: {}", + in.getFileName(), distributed_header.revision, DBMS_TCP_PROTOCOL_VERSION); + } + + if (header_buf.hasPendingData()) + { + readVarUInt(distributed_header.shard_num, header_buf); + readStringBinary(distributed_header.cluster, header_buf); + readStringBinary(distributed_header.distributed_table, header_buf); + readStringBinary(distributed_header.remote_table, header_buf); + } + + /// Add handling new data here, for example: + /// + /// if (header_buf.hasPendingData()) + /// readVarUInt(my_new_data, header_buf); + /// + /// And note that it is safe, because we have checksum and size for header. + + return distributed_header; + } + + if (query_size == DBMS_DISTRIBUTED_SIGNATURE_HEADER_OLD_FORMAT) + { + distributed_header.insert_settings.read(in, SettingsWriteFormat::BINARY); + readStringBinary(distributed_header.insert_query, in); + return distributed_header; + } + + distributed_header.insert_query.resize(query_size); + in.readStrict(distributed_header.insert_query.data(), query_size); + + return distributed_header; +} + +OpenTelemetry::TracingContextHolderPtr DistributedAsyncInsertHeader::createTracingContextHolder(const char * function, std::shared_ptr open_telemetry_span_log) const +{ + OpenTelemetry::TracingContextHolderPtr trace_context = std::make_unique( + function, + client_info.client_trace_context, + std::move(open_telemetry_span_log)); + trace_context->root_span.addAttribute("clickhouse.shard_num", shard_num); + trace_context->root_span.addAttribute("clickhouse.cluster", cluster); + trace_context->root_span.addAttribute("clickhouse.distributed", distributed_table); + trace_context->root_span.addAttribute("clickhouse.remote", remote_table); + trace_context->root_span.addAttribute("clickhouse.rows", rows); + trace_context->root_span.addAttribute("clickhouse.bytes", bytes); + return trace_context; +} + +} diff --git a/src/Storages/Distributed/DistributedAsyncInsertHeader.h b/src/Storages/Distributed/DistributedAsyncInsertHeader.h new file mode 100644 index 00000000000..a7330fa5ef1 --- /dev/null +++ b/src/Storages/Distributed/DistributedAsyncInsertHeader.h @@ -0,0 +1,45 @@ +#pragma once + +#include +#include +#include +#include +#include + +namespace DB +{ + +class ReadBufferFromFile; + +namespace OpenTelemetry +{ +struct TracingContextHolder; +using TracingContextHolderPtr = std::unique_ptr; +} + +/// Header for the binary files that are stored on disk for async INSERT into Distributed. +struct DistributedAsyncInsertHeader +{ + UInt64 revision = 0; + Settings insert_settings; + std::string insert_query; + ClientInfo client_info; + + /// .bin file cannot have zero rows/bytes. + size_t rows = 0; + size_t bytes = 0; + + UInt32 shard_num = 0; + std::string cluster; + std::string distributed_table; + std::string remote_table; + + /// dumpStructure() of the header -- obsolete + std::string block_header_string; + Block block_header; + + static DistributedAsyncInsertHeader read(ReadBufferFromFile & in, Poco::Logger * log); + OpenTelemetry::TracingContextHolderPtr createTracingContextHolder(const char * function, std::shared_ptr open_telemetry_span_log) const; +}; + +} diff --git a/src/Storages/Distributed/DistributedAsyncInsertHelpers.cpp b/src/Storages/Distributed/DistributedAsyncInsertHelpers.cpp new file mode 100644 index 00000000000..98073ba1e08 --- /dev/null +++ b/src/Storages/Distributed/DistributedAsyncInsertHelpers.cpp @@ -0,0 +1,124 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int CANNOT_READ_ALL_DATA; + extern const int UNKNOWN_CODEC; + extern const int CANNOT_DECOMPRESS; + extern const int CHECKSUM_DOESNT_MATCH; + extern const int TOO_LARGE_SIZE_COMPRESSED; + extern const int ATTEMPT_TO_READ_AFTER_EOF; + extern const int EMPTY_DATA_PASSED; + extern const int DISTRIBUTED_BROKEN_BATCH_INFO; + extern const int DISTRIBUTED_BROKEN_BATCH_FILES; +} + +/// 'remote_error' argument is used to decide whether some errors should be +/// ignored or not, in particular: +/// +/// - ATTEMPT_TO_READ_AFTER_EOF should not be ignored +/// if we receive it from remote (receiver), since: +/// - the sender will got ATTEMPT_TO_READ_AFTER_EOF when the client just go away, +/// i.e. server had been restarted +/// - since #18853 the file will be checked on the sender locally, and +/// if there is something wrong with the file itself, we will receive +/// ATTEMPT_TO_READ_AFTER_EOF not from the remote at first +/// and mark batch as broken. +bool isDistributedSendBroken(int code, bool remote_error) +{ + return code == ErrorCodes::CHECKSUM_DOESNT_MATCH + || code == ErrorCodes::EMPTY_DATA_PASSED + || code == ErrorCodes::TOO_LARGE_SIZE_COMPRESSED + || code == ErrorCodes::CANNOT_READ_ALL_DATA + || code == ErrorCodes::UNKNOWN_CODEC + || code == ErrorCodes::CANNOT_DECOMPRESS + || code == ErrorCodes::DISTRIBUTED_BROKEN_BATCH_INFO + || code == ErrorCodes::DISTRIBUTED_BROKEN_BATCH_FILES + || (!remote_error && code == ErrorCodes::ATTEMPT_TO_READ_AFTER_EOF); +} + +void writeAndConvert(RemoteInserter & remote, const DistributedAsyncInsertHeader & distributed_header, ReadBufferFromFile & in) +{ + CompressedReadBuffer decompressing_in(in); + NativeReader block_in(decompressing_in, distributed_header.revision); + + while (Block block = block_in.read()) + { + auto converting_dag = ActionsDAG::makeConvertingActions( + block.cloneEmpty().getColumnsWithTypeAndName(), + remote.getHeader().getColumnsWithTypeAndName(), + ActionsDAG::MatchColumnsMode::Name); + + auto converting_actions = std::make_shared(std::move(converting_dag)); + converting_actions->execute(block); + remote.write(block); + } +} + +void writeRemoteConvert( + const DistributedAsyncInsertHeader & distributed_header, + RemoteInserter & remote, + bool compression_expected, + ReadBufferFromFile & in, + Poco::Logger * log) +{ + if (!remote.getHeader()) + { + CheckingCompressedReadBuffer checking_in(in); + remote.writePrepared(checking_in); + return; + } + + /// This is old format, that does not have header for the block in the file header, + /// applying ConvertingTransform in this case is not a big overhead. + /// + /// Anyway we can get header only from the first block, which contain all rows anyway. + if (!distributed_header.block_header) + { + LOG_TRACE(log, "Processing batch {} with old format (no header)", in.getFileName()); + + writeAndConvert(remote, distributed_header, in); + return; + } + + if (!blocksHaveEqualStructure(distributed_header.block_header, remote.getHeader())) + { + LOG_WARNING(log, + "Structure does not match (remote: {}, local: {}), implicit conversion will be done", + remote.getHeader().dumpStructure(), distributed_header.block_header.dumpStructure()); + + writeAndConvert(remote, distributed_header, in); + return; + } + + /// If connection does not use compression, we have to uncompress the data. + if (!compression_expected) + { + writeAndConvert(remote, distributed_header, in); + return; + } + + if (distributed_header.revision != remote.getServerRevision()) + { + writeAndConvert(remote, distributed_header, in); + return; + } + + /// Otherwise write data as it was already prepared (more efficient path). + CheckingCompressedReadBuffer checking_in(in); + remote.writePrepared(checking_in); +} + +} diff --git a/src/Storages/Distributed/DistributedAsyncInsertHelpers.h b/src/Storages/Distributed/DistributedAsyncInsertHelpers.h new file mode 100644 index 00000000000..9543450418c --- /dev/null +++ b/src/Storages/Distributed/DistributedAsyncInsertHelpers.h @@ -0,0 +1,35 @@ +#pragma once + +namespace Poco +{ +class Logger; +} + +namespace DB +{ + +struct DistributedAsyncInsertHeader; +class ReadBufferFromFile; +class RemoteInserter; + +/// 'remote_error' argument is used to decide whether some errors should be +/// ignored or not, in particular: +/// +/// - ATTEMPT_TO_READ_AFTER_EOF should not be ignored +/// if we receive it from remote (receiver), since: +/// - the sender will got ATTEMPT_TO_READ_AFTER_EOF when the client just go away, +/// i.e. server had been restarted +/// - since #18853 the file will be checked on the sender locally, and +/// if there is something wrong with the file itself, we will receive +/// ATTEMPT_TO_READ_AFTER_EOF not from the remote at first +/// and mark batch as broken. +bool isDistributedSendBroken(int code, bool remote_error); + +void writeRemoteConvert( + const DistributedAsyncInsertHeader & distributed_header, + RemoteInserter & remote, + bool compression_expected, + ReadBufferFromFile & in, + Poco::Logger * log); + +} diff --git a/src/Storages/Distributed/DistributedAsyncInsertSource.cpp b/src/Storages/Distributed/DistributedAsyncInsertSource.cpp new file mode 100644 index 00000000000..7992636ac11 --- /dev/null +++ b/src/Storages/Distributed/DistributedAsyncInsertSource.cpp @@ -0,0 +1,62 @@ +#include +#include +#include +#include +#include +#include + +namespace DB +{ + +struct DistributedAsyncInsertSource::Data +{ + Poco::Logger * log = nullptr; + + ReadBufferFromFile in; + CompressedReadBuffer decompressing_in; + NativeReader block_in; + + Block first_block; + + explicit Data(const String & file_name) + : log(&Poco::Logger::get("DistributedAsyncInsertSource")) + , in(file_name) + , decompressing_in(in) + , block_in(decompressing_in, DistributedAsyncInsertHeader::read(in, log).revision) + , first_block(block_in.read()) + { + } +}; + +DistributedAsyncInsertSource::DistributedAsyncInsertSource(const String & file_name) + : DistributedAsyncInsertSource(std::make_unique(file_name)) +{ +} + +DistributedAsyncInsertSource::DistributedAsyncInsertSource(std::unique_ptr data_) + : ISource(data_->first_block.cloneEmpty()) + , data(std::move(data_)) +{ +} + +DistributedAsyncInsertSource::~DistributedAsyncInsertSource() = default; + +Chunk DistributedAsyncInsertSource::generate() +{ + if (data->first_block) + { + size_t num_rows = data->first_block.rows(); + Chunk res(data->first_block.getColumns(), num_rows); + data->first_block.clear(); + return res; + } + + auto block = data->block_in.read(); + if (!block) + return {}; + + size_t num_rows = block.rows(); + return Chunk(block.getColumns(), num_rows); +} + +} diff --git a/src/Storages/Distributed/DistributedAsyncInsertSource.h b/src/Storages/Distributed/DistributedAsyncInsertSource.h new file mode 100644 index 00000000000..35f846151dc --- /dev/null +++ b/src/Storages/Distributed/DistributedAsyncInsertSource.h @@ -0,0 +1,28 @@ +#pragma once + +#include +#include +#include + +namespace DB +{ + +/// Source for the Distributed engine on-disk file for async INSERT. +class DistributedAsyncInsertSource : public ISource +{ + struct Data; + explicit DistributedAsyncInsertSource(std::unique_ptr data); + +public: + explicit DistributedAsyncInsertSource(const String & file_name); + ~DistributedAsyncInsertSource() override; + String getName() const override { return "DistributedAsyncInsertSource"; } + +protected: + Chunk generate() override; + +private: + std::unique_ptr data; +}; + +} diff --git a/src/Storages/Distributed/DistributedSink.cpp b/src/Storages/Distributed/DistributedSink.cpp index bac13ea37cf..d388a403031 100644 --- a/src/Storages/Distributed/DistributedSink.cpp +++ b/src/Storages/Distributed/DistributedSink.cpp @@ -1,5 +1,5 @@ #include -#include +#include #include #include #include @@ -340,9 +340,9 @@ DistributedSink::runWritingJob(JobReplica & job, const Block & current_block, si size_t rows = shard_block.rows(); span.addAttribute("clickhouse.shard_num", shard_info.shard_num); - span.addAttribute("clickhouse.cluster", this->storage.cluster_name); - span.addAttribute("clickhouse.distributed", this->storage.getStorageID().getFullNameNotQuoted()); - span.addAttribute("clickhouse.remote", [this]() { return storage.remote_database + "." + storage.remote_table; }); + span.addAttribute("clickhouse.cluster", storage.cluster_name); + span.addAttribute("clickhouse.distributed", storage.getStorageID().getFullNameNotQuoted()); + span.addAttribute("clickhouse.remote", [this]() { return storage.getRemoteDatabaseName() + "." + storage.getRemoteTableName(); }); span.addAttribute("clickhouse.rows", rows); span.addAttribute("clickhouse.bytes", [&shard_block]() { return toString(shard_block.bytes()); }); @@ -476,7 +476,7 @@ void DistributedSink::writeSync(const Block & block) span.addAttribute("clickhouse.start_shard", start); span.addAttribute("clickhouse.end_shard", end); - span.addAttribute("db.statement", this->query_string); + span.addAttribute("db.statement", query_string); if (num_shards > 1) { @@ -569,6 +569,26 @@ void DistributedSink::onFinish() } } +void DistributedSink::onCancel() +{ + if (pool && !pool->finished()) + { + try + { + pool->wait(); + } + catch (...) + { + tryLogCurrentException(storage.log); + } + } + + for (auto & shard_jobs : per_shard_jobs) + for (JobReplica & job : shard_jobs.replicas_jobs) + if (job.executor) + job.executor->cancel(); +} + IColumn::Selector DistributedSink::createSelector(const Block & source_block) const { @@ -659,9 +679,9 @@ void DistributedSink::writeToLocal(const Cluster::ShardInfo & shard_info, const { OpenTelemetry::SpanHolder span(__PRETTY_FUNCTION__); span.addAttribute("clickhouse.shard_num", shard_info.shard_num); - span.addAttribute("clickhouse.cluster", this->storage.cluster_name); - span.addAttribute("clickhouse.distributed", this->storage.getStorageID().getFullNameNotQuoted()); - span.addAttribute("clickhouse.remote", [this]() { return storage.remote_database + "." + storage.remote_table; }); + span.addAttribute("clickhouse.cluster", storage.cluster_name); + span.addAttribute("clickhouse.distributed", storage.getStorageID().getFullNameNotQuoted()); + span.addAttribute("clickhouse.remote", [this]() { return storage.getRemoteDatabaseName() + "." + storage.getRemoteTableName(); }); span.addAttribute("clickhouse.rows", [&block]() { return toString(block.rows()); }); span.addAttribute("clickhouse.bytes", [&block]() { return toString(block.bytes()); }); @@ -705,7 +725,7 @@ void DistributedSink::writeToShard(const Cluster::ShardInfo & shard_info, const CompressionCodecPtr compression_codec = CompressionCodecFactory::instance().get(compression_method, compression_level); /// tmp directory is used to ensure atomicity of transactions - /// and keep monitor thread out from reading incomplete data + /// and keep directory queue thread out from reading incomplete data std::string first_file_tmp_path; auto reservation = storage.getStoragePolicy()->reserveAndCheck(block.bytes()); @@ -724,6 +744,9 @@ void DistributedSink::writeToShard(const Cluster::ShardInfo & shard_info, const return guard; }; + auto sleep_ms = context->getSettingsRef().distributed_directory_monitor_sleep_time_ms.totalMilliseconds(); + size_t file_size; + auto it = dir_names.begin(); /// on first iteration write block to a temporary directory for subsequent /// hardlinking to ensure the inode is not freed until we're done @@ -779,9 +802,9 @@ void DistributedSink::writeToShard(const Cluster::ShardInfo & shard_info, const } writeVarUInt(shard_info.shard_num, header_buf); - writeStringBinary(this->storage.cluster_name, header_buf); - writeStringBinary(this->storage.getStorageID().getFullNameNotQuoted(), header_buf); - writeStringBinary(this->storage.remote_database + "." + this->storage.remote_table, header_buf); + writeStringBinary(storage.cluster_name, header_buf); + writeStringBinary(storage.getStorageID().getFullNameNotQuoted(), header_buf); + writeStringBinary(storage.getRemoteDatabaseName() + "." + storage.getRemoteTableName(), header_buf); /// Add new fields here, for example: /// writeVarUInt(my_new_data, header_buf); @@ -801,10 +824,16 @@ void DistributedSink::writeToShard(const Cluster::ShardInfo & shard_info, const out.sync(); } + file_size = fs::file_size(first_file_tmp_path); + // Create hardlink here to reuse increment number - const std::string block_file_path(fs::path(path) / file_name); - createHardLink(first_file_tmp_path, block_file_path); - auto dir_sync_guard = make_directory_sync_guard(*it); + auto bin_file = (fs::path(path) / file_name).string(); + auto & directory_queue = storage.getDirectoryQueue(disk, *it); + { + createHardLink(first_file_tmp_path, bin_file); + auto dir_sync_guard = make_directory_sync_guard(*it); + } + directory_queue.addFileAndSchedule(bin_file, file_size, sleep_ms); } ++it; @@ -814,23 +843,18 @@ void DistributedSink::writeToShard(const Cluster::ShardInfo & shard_info, const const std::string path(fs::path(disk_path) / (data_path + *it)); fs::create_directory(path); - const std::string block_file_path(fs::path(path) / (toString(storage.file_names_increment.get()) + ".bin")); - createHardLink(first_file_tmp_path, block_file_path); - auto dir_sync_guard = make_directory_sync_guard(*it); + auto bin_file = (fs::path(path) / (toString(storage.file_names_increment.get()) + ".bin")).string(); + auto & directory_queue = storage.getDirectoryQueue(disk, *it); + { + createHardLink(first_file_tmp_path, bin_file); + auto dir_sync_guard = make_directory_sync_guard(*it); + } + directory_queue.addFileAndSchedule(bin_file, file_size, sleep_ms); } - auto file_size = fs::file_size(first_file_tmp_path); /// remove the temporary file, enabling the OS to reclaim inode after all threads /// have removed their corresponding files fs::remove(first_file_tmp_path); - - /// Notify - auto sleep_ms = context->getSettingsRef().distributed_directory_monitor_sleep_time_ms; - for (const auto & dir_name : dir_names) - { - auto & directory_monitor = storage.requireDirectoryMonitor(disk, dir_name); - directory_monitor.addAndSchedule(file_size, sleep_ms.totalMilliseconds()); - } } } diff --git a/src/Storages/Distributed/DistributedSink.h b/src/Storages/Distributed/DistributedSink.h index af0c64cbd78..325d5859289 100644 --- a/src/Storages/Distributed/DistributedSink.h +++ b/src/Storages/Distributed/DistributedSink.h @@ -54,6 +54,8 @@ public: void onFinish() override; private: + void onCancel() override; + IColumn::Selector createSelector(const Block & source_block) const; void writeAsync(const Block & block); diff --git a/src/Storages/HDFS/StorageHDFSCluster.cpp b/src/Storages/HDFS/StorageHDFSCluster.cpp index 8dbaa0796e9..19af38eff11 100644 --- a/src/Storages/HDFS/StorageHDFSCluster.cpp +++ b/src/Storages/HDFS/StorageHDFSCluster.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -83,8 +84,12 @@ Pipe StorageHDFSCluster::read( auto extension = getTaskIteratorExtension(query_info.query, context); /// Calculate the header. This is significant, because some columns could be thrown away in some cases like query with count(*) - Block header = - InterpreterSelectQuery(query_info.query, context, SelectQueryOptions(processed_stage).analyze()).getSampleBlock(); + Block header; + + if (context->getSettingsRef().allow_experimental_analyzer) + header = InterpreterSelectQueryAnalyzer::getSampleBlock(query_info.query, context, SelectQueryOptions(processed_stage).analyze()); + else + header = InterpreterSelectQuery(query_info.query, context, SelectQueryOptions(processed_stage).analyze()).getSampleBlock(); const Scalars & scalars = context->hasQueryContext() ? context->getQueryContext()->getScalars() : Scalars{}; @@ -105,7 +110,6 @@ Pipe StorageHDFSCluster::read( for (auto & try_result : try_results) { auto remote_query_executor = std::make_shared( - shard_info.pool, std::vector{try_result}, queryToString(query_to_send), header, diff --git a/src/Storages/IStorage.cpp b/src/Storages/IStorage.cpp index 9bcfff65c95..d50f335c1c9 100644 --- a/src/Storages/IStorage.cpp +++ b/src/Storages/IStorage.cpp @@ -40,7 +40,7 @@ RWLockImpl::LockHolder IStorage::tryLockTimed( { const String type_str = type == RWLockImpl::Type::Read ? "READ" : "WRITE"; throw Exception(ErrorCodes::DEADLOCK_AVOIDED, - "{} locking attempt on \"{}\" has timed out! ({}ms) Possible deadlock avoided. Client should retry.", + "{} locking attempt on \"{}\" has timed out! ({}ms) Possible deadlock avoided. Client should retry", type_str, getStorageID(), acquire_timeout.count()); } return lock_holder; diff --git a/src/Storages/IndicesDescription.cpp b/src/Storages/IndicesDescription.cpp index 2e07aceeaa9..a93ac248c98 100644 --- a/src/Storages/IndicesDescription.cpp +++ b/src/Storages/IndicesDescription.cpp @@ -94,14 +94,15 @@ IndexDescription IndexDescription::getIndexFromAST(const ASTPtr & definition_ast auto syntax = TreeRewriter(context).analyze(expr_list, columns.getAllPhysical()); result.expression = ExpressionAnalyzer(expr_list, syntax, context).getActions(true); - Block block_without_columns = result.expression->getSampleBlock(); + result.sample_block = result.expression->getSampleBlock(); - for (size_t i = 0; i < block_without_columns.columns(); ++i) + for (auto & elem : result.sample_block) { - const auto & column = block_without_columns.getByPosition(i); - result.column_names.emplace_back(column.name); - result.data_types.emplace_back(column.type); - result.sample_block.insert(ColumnWithTypeAndName(column.type->createColumn(), column.type, column.name)); + if (!elem.column) + elem.column = elem.type->createColumn(); + + result.column_names.push_back(elem.name); + result.data_types.push_back(elem.type); } const auto & definition_arguments = index_definition->type->arguments; diff --git a/src/Storages/Kafka/StorageKafka.cpp b/src/Storages/Kafka/StorageKafka.cpp index 61495c966cb..538b170ca59 100644 --- a/src/Storages/Kafka/StorageKafka.cpp +++ b/src/Storages/Kafka/StorageKafka.cpp @@ -968,6 +968,11 @@ void registerStorageKafka(StorageFactory & factory) { throw Exception(ErrorCodes::BAD_ARGUMENTS, "kafka_poll_max_batch_size can not be lower than 1"); } + if (args.columns.getOrdinary() != args.columns.getAll() || !args.columns.getWithDefaultExpression().empty()) + { + throw Exception(ErrorCodes::BAD_ARGUMENTS, "KafkaEngine doesn't support DEFAULT/MATERIALIZED/EPHEMERAL/ALIAS expressions for columns. " + "See https://clickhouse.com/docs/en/engines/table-engines/integrations/kafka/#configuration"); + } return std::make_shared(args.table_id, args.getContext(), args.columns, std::move(kafka_settings), collection_name); }; diff --git a/src/Storages/LiveView/LiveViewSink.h b/src/Storages/LiveView/LiveViewSink.h index 1d90e35618f..e163400f2af 100644 --- a/src/Storages/LiveView/LiveViewSink.h +++ b/src/Storages/LiveView/LiveViewSink.h @@ -3,7 +3,7 @@ #include #include #include -#include +#include namespace DB diff --git a/src/Storages/LiveView/StorageLiveView.cpp b/src/Storages/LiveView/StorageLiveView.cpp index 547becf3837..2c3e452de92 100644 --- a/src/Storages/LiveView/StorageLiveView.cpp +++ b/src/Storages/LiveView/StorageLiveView.cpp @@ -27,7 +27,7 @@ limitations under the License. */ #include #include #include -#include +#include #include #include @@ -241,7 +241,18 @@ StorageLiveView::StorageLiveView( blocks_metadata_ptr = std::make_shared(); active_ptr = std::make_shared(true); - periodic_refresh_task = getContext()->getSchedulePool().createTask("LiveViewPeriodicRefreshTask", [this]{ periodicRefreshTaskFunc(); }); + periodic_refresh_task = getContext()->getSchedulePool().createTask("LiveViewPeriodicRefreshTask", + [this] + { + try + { + periodicRefreshTaskFunc(); + } + catch (...) + { + tryLogCurrentException(log, "Exception in LiveView periodic refresh task in BackgroundSchedulePool"); + } + }); periodic_refresh_task->deactivate(); } diff --git a/src/Storages/MarkCache.h b/src/Storages/MarkCache.h index 9095bf6bb35..f9272b1f4bb 100644 --- a/src/Storages/MarkCache.h +++ b/src/Storages/MarkCache.h @@ -26,7 +26,7 @@ struct MarksWeightFunction size_t operator()(const MarksInCompressedFile & marks) const { - return marks.size() * sizeof(MarkInCompressedFile) + MARK_CACHE_OVERHEAD; + return marks.approximateMemoryUsage() + MARK_CACHE_OVERHEAD; } }; diff --git a/src/Storages/MergeTree/MergeFromLogEntryTask.cpp b/src/Storages/MergeTree/MergeFromLogEntryTask.cpp index e017c9681e8..7e5d0a2d643 100644 --- a/src/Storages/MergeTree/MergeFromLogEntryTask.cpp +++ b/src/Storages/MergeTree/MergeFromLogEntryTask.cpp @@ -218,7 +218,7 @@ ReplicatedMergeMutateTaskBase::PrepareResult MergeFromLogEntryTask::prepare() zero_copy_lock = storage.tryCreateZeroCopyExclusiveLock(entry.new_part_name, disk); - if (!zero_copy_lock) + if (!zero_copy_lock || !zero_copy_lock->isLocked()) { LOG_DEBUG(log, "Merge of part {} started by some other replica, will wait it and fetch merged part", entry.new_part_name); /// Don't check for missing part -- it's missing because other replica still not diff --git a/src/Storages/MergeTree/MergeList.cpp b/src/Storages/MergeTree/MergeList.cpp index 76d69cc6b7d..fa1887a02e6 100644 --- a/src/Storages/MergeTree/MergeList.cpp +++ b/src/Storages/MergeTree/MergeList.cpp @@ -144,8 +144,11 @@ MergeInfo MergeListElement::getInfo() const MergeListElement::~MergeListElement() { - CurrentThread::getMemoryTracker()->adjustWithUntrackedMemory(untracked_memory); - untracked_memory = 0; + if (untracked_memory != 0) + { + CurrentThread::getMemoryTracker()->adjustWithUntrackedMemory(untracked_memory); + untracked_memory = 0; + } } diff --git a/src/Storages/MergeTree/MergeTask.cpp b/src/Storages/MergeTree/MergeTask.cpp index b961b70428e..9d9d8420e2c 100644 --- a/src/Storages/MergeTree/MergeTask.cpp +++ b/src/Storages/MergeTree/MergeTask.cpp @@ -96,6 +96,32 @@ static void extractMergingAndGatheringColumns( } } +static void addMissedColumnsToSerializationInfos( + size_t num_rows_in_parts, + const Names & part_columns, + const ColumnsDescription & storage_columns, + const SerializationInfo::Settings & info_settings, + SerializationInfoByName & new_infos) +{ + NameSet part_columns_set(part_columns.begin(), part_columns.end()); + + for (const auto & column : storage_columns) + { + if (part_columns_set.contains(column.name)) + continue; + + if (column.default_desc.kind != ColumnDefaultKind::Default) + continue; + + if (column.default_desc.expression) + continue; + + auto new_info = column.type->createSerializationInfo(info_settings); + new_info->addDefaults(num_rows_in_parts); + new_infos.emplace(column.name, std::move(new_info)); + } +} + bool MergeTask::ExecuteAndFinalizeHorizontalPart::prepare() { @@ -205,7 +231,19 @@ bool MergeTask::ExecuteAndFinalizeHorizontalPart::prepare() ctx->force_ttl = true; } - infos.add(part->getSerializationInfos()); + if (!info_settings.isAlwaysDefault()) + { + auto part_infos = part->getSerializationInfos(); + + addMissedColumnsToSerializationInfos( + part->rows_count, + part->getColumns().getNames(), + global_ctx->metadata_snapshot->getColumns(), + info_settings, + part_infos); + + infos.add(part_infos); + } } global_ctx->new_data_part->setColumns(global_ctx->storage_columns, infos); diff --git a/src/Storages/MergeTree/MergeTreeBackgroundExecutor.cpp b/src/Storages/MergeTree/MergeTreeBackgroundExecutor.cpp index d0469c35cef..84fa9ec2c8e 100644 --- a/src/Storages/MergeTree/MergeTreeBackgroundExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeBackgroundExecutor.cpp @@ -59,6 +59,7 @@ void MergeTreeBackgroundExecutor::increaseThreadsAndMaxTasksCount(size_t for (size_t number = threads_count; number < new_threads_count; ++number) pool.scheduleOrThrowOnError([this] { threadFunction(); }); + max_tasks_metric.changeTo(2 * new_max_tasks_count); // pending + active max_tasks_count.store(new_max_tasks_count, std::memory_order_relaxed); threads_count = new_threads_count; } diff --git a/src/Storages/MergeTree/MergeTreeBackgroundExecutor.h b/src/Storages/MergeTree/MergeTreeBackgroundExecutor.h index 9305f36feb5..5c47d20865b 100644 --- a/src/Storages/MergeTree/MergeTreeBackgroundExecutor.h +++ b/src/Storages/MergeTree/MergeTreeBackgroundExecutor.h @@ -13,6 +13,7 @@ #include #include +#include #include #include #include @@ -247,11 +248,13 @@ public: String name_, size_t threads_count_, size_t max_tasks_count_, - CurrentMetrics::Metric metric_) + CurrentMetrics::Metric metric_, + CurrentMetrics::Metric max_tasks_metric_) : name(name_) , threads_count(threads_count_) , max_tasks_count(max_tasks_count_) , metric(metric_) + , max_tasks_metric(max_tasks_metric_, 2 * max_tasks_count) // active + pending { if (max_tasks_count == 0) throw Exception(ErrorCodes::INVALID_CONFIG_PARAMETER, "Task count for MergeTreeBackgroundExecutor must not be zero"); @@ -272,9 +275,10 @@ public: size_t threads_count_, size_t max_tasks_count_, CurrentMetrics::Metric metric_, + CurrentMetrics::Metric max_tasks_metric_, std::string_view policy) requires requires(Queue queue) { queue.updatePolicy(policy); } // Because we use explicit template instantiation - : MergeTreeBackgroundExecutor(name_, threads_count_, max_tasks_count_, metric_) + : MergeTreeBackgroundExecutor(name_, threads_count_, max_tasks_count_, metric_, max_tasks_metric_) { pending.updatePolicy(policy); } @@ -311,6 +315,7 @@ private: size_t threads_count TSA_GUARDED_BY(mutex) = 0; std::atomic max_tasks_count = 0; CurrentMetrics::Metric metric; + CurrentMetrics::Increment max_tasks_metric; void routine(TaskRuntimeDataPtr item); diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 1da99cb4117..e2dc048a0e8 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -525,7 +525,6 @@ void MergeTreeData::checkProperties( for (const auto & index : new_metadata.secondary_indices) { - MergeTreeIndexFactory::instance().validate(index, attach); if (indices_names.find(index.name) != indices_names.end()) @@ -1948,9 +1947,9 @@ size_t MergeTreeData::clearOldTemporaryDirectories(size_t custom_directories_lif { if (temporary_parts.contains(basename)) { - /// Actually we don't rely on temporary_directories_lifetime when removing old temporaries directoties, + /// Actually we don't rely on temporary_directories_lifetime when removing old temporaries directories, /// it's just an extra level of protection just in case we have a bug. - LOG_INFO(log, "{} is in use (by merge/mutation/INSERT) (consider increasing temporary_directories_lifetime setting)", full_path); + LOG_INFO(LogFrequencyLimiter(log, 10), "{} is in use (by merge/mutation/INSERT) (consider increasing temporary_directories_lifetime setting)", full_path); continue; } else @@ -4122,9 +4121,9 @@ void MergeTreeData::delayInsertOrThrowIfNeeded(Poco::Event * until, const Contex ProfileEvents::increment(ProfileEvents::RejectedInserts); throw Exception( ErrorCodes::TOO_MANY_PARTS, - "Too many parts ({}) in all partitions in total. This indicates wrong choice of partition key. The threshold can be modified " + "Too many parts ({}) in all partitions in total in table '{}'. This indicates wrong choice of partition key. The threshold can be modified " "with 'max_parts_in_total' setting in element in config.xml or with per-table setting.", - parts_count_in_total); + parts_count_in_total, getLogName()); } size_t outdated_parts_over_threshold = 0; @@ -4138,8 +4137,8 @@ void MergeTreeData::delayInsertOrThrowIfNeeded(Poco::Event * until, const Contex ProfileEvents::increment(ProfileEvents::RejectedInserts); throw Exception( ErrorCodes::TOO_MANY_PARTS, - "Too many inactive parts ({}). Parts cleaning are processing significantly slower than inserts", - outdated_parts_count_in_partition); + "Too many inactive parts ({}) in table '{}'. Parts cleaning are processing significantly slower than inserts", + outdated_parts_count_in_partition, getLogName()); } if (settings->inactive_parts_to_delay_insert > 0 && outdated_parts_count_in_partition >= settings->inactive_parts_to_delay_insert) outdated_parts_over_threshold = outdated_parts_count_in_partition - settings->inactive_parts_to_delay_insert + 1; @@ -4152,6 +4151,7 @@ void MergeTreeData::delayInsertOrThrowIfNeeded(Poco::Event * until, const Contex const auto active_parts_to_throw_insert = query_settings.parts_to_throw_insert ? query_settings.parts_to_throw_insert : settings->parts_to_throw_insert; size_t active_parts_over_threshold = 0; + { bool parts_are_large_enough_in_average = settings->max_avg_part_size_for_too_many_parts && average_part_size > settings->max_avg_part_size_for_too_many_parts; @@ -4161,9 +4161,10 @@ void MergeTreeData::delayInsertOrThrowIfNeeded(Poco::Event * until, const Contex ProfileEvents::increment(ProfileEvents::RejectedInserts); throw Exception( ErrorCodes::TOO_MANY_PARTS, - "Too many parts ({} with average size of {}). Merges are processing significantly slower than inserts", + "Too many parts ({} with average size of {}) in table '{}'. Merges are processing significantly slower than inserts", parts_count_in_partition, - ReadableSize(average_part_size)); + ReadableSize(average_part_size), + getLogName()); } if (active_parts_to_delay_insert > 0 && parts_count_in_partition >= active_parts_to_delay_insert && !parts_are_large_enough_in_average) @@ -5081,12 +5082,8 @@ void MergeTreeData::restorePartFromBackup(std::shared_ptr r if (filename.ends_with(IMergeTreeDataPart::TXN_VERSION_METADATA_FILE_NAME)) continue; - auto backup_entry = backup->readFile(part_path_in_backup_fs / filename); - auto read_buffer = backup_entry->getReadBuffer(); - auto write_buffer = disk->writeFile(temp_part_dir / filename); - copyData(*read_buffer, *write_buffer); - write_buffer->finalize(); - reservation->update(reservation->getSize() - backup_entry->getSize()); + size_t file_size = backup->copyFileToDisk(part_path_in_backup_fs / filename, disk, temp_part_dir / filename); + reservation->update(reservation->getSize() - file_size); } auto single_disk_volume = std::make_shared(disk->getName(), disk, 0); @@ -7487,7 +7484,7 @@ MovePartsOutcome MergeTreeData::movePartsToSpace(const DataPartsVector & parts, if (moving_tagger->parts_to_move.empty()) return MovePartsOutcome::NothingToMove; - return moveParts(moving_tagger); + return moveParts(moving_tagger, true); } MergeTreeData::CurrentlyMovingPartsTaggerPtr MergeTreeData::selectPartsForMove() @@ -7542,7 +7539,7 @@ MergeTreeData::CurrentlyMovingPartsTaggerPtr MergeTreeData::checkPartsForMove(co return std::make_shared(std::move(parts_to_move), *this); } -MovePartsOutcome MergeTreeData::moveParts(const CurrentlyMovingPartsTaggerPtr & moving_tagger) +MovePartsOutcome MergeTreeData::moveParts(const CurrentlyMovingPartsTaggerPtr & moving_tagger, bool wait_for_move_if_zero_copy) { LOG_INFO(log, "Got {} parts to move.", moving_tagger->parts_to_move.size()); @@ -7591,21 +7588,41 @@ MovePartsOutcome MergeTreeData::moveParts(const CurrentlyMovingPartsTaggerPtr & auto disk = moving_part.reserved_space->getDisk(); if (supportsReplication() && disk->supportZeroCopyReplication() && settings->allow_remote_fs_zero_copy_replication) { - /// If we acquired lock than let's try to move. After one - /// replica will actually move the part from disk to some - /// zero-copy storage other replicas will just fetch - /// metainformation. - if (auto lock = tryCreateZeroCopyExclusiveLock(moving_part.part->name, disk); lock) + /// This loop is not endless, if shutdown called/connection failed/replica became readonly + /// we will return true from waitZeroCopyLock and createZeroCopyLock will return nullopt. + while (true) { - cloned_part = parts_mover.clonePart(moving_part); - parts_mover.swapClonedPart(cloned_part); - } - else - { - /// Move will be retried but with backoff. - LOG_DEBUG(log, "Move of part {} postponed, because zero copy mode enabled and someone other moving this part right now", moving_part.part->name); - result = MovePartsOutcome::MoveWasPostponedBecauseOfZeroCopy; - continue; + /// If we acquired lock than let's try to move. After one + /// replica will actually move the part from disk to some + /// zero-copy storage other replicas will just fetch + /// metainformation. + if (auto lock = tryCreateZeroCopyExclusiveLock(moving_part.part->name, disk); lock) + { + if (lock->isLocked()) + { + cloned_part = parts_mover.clonePart(moving_part); + parts_mover.swapClonedPart(cloned_part); + break; + } + else if (wait_for_move_if_zero_copy) + { + LOG_DEBUG(log, "Other replica is working on move of {}, will wait until lock disappear", moving_part.part->name); + /// Wait and checks not only for timeout but also for shutdown and so on. + while (!waitZeroCopyLockToDisappear(*lock, 3000)) + { + LOG_DEBUG(log, "Waiting until some replica will move {} and zero copy lock disappear", moving_part.part->name); + } + } + else + break; + } + else + { + /// Move will be retried but with backoff. + LOG_DEBUG(log, "Move of part {} postponed, because zero copy mode enabled and someone other moving this part right now", moving_part.part->name); + result = MovePartsOutcome::MoveWasPostponedBecauseOfZeroCopy; + break; + } } } else /// Ordinary move as it should be diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index 4a1aafe20b6..bc5e5bc2d91 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -1456,7 +1456,7 @@ private: using CurrentlyMovingPartsTaggerPtr = std::shared_ptr; /// Move selected parts to corresponding disks - MovePartsOutcome moveParts(const CurrentlyMovingPartsTaggerPtr & moving_tagger); + MovePartsOutcome moveParts(const CurrentlyMovingPartsTaggerPtr & moving_tagger, bool wait_for_move_if_zero_copy=false); /// Select parts for move and disks for them. Used in background moving processes. CurrentlyMovingPartsTaggerPtr selectPartsForMove(); @@ -1511,6 +1511,7 @@ private: /// Create zero-copy exclusive lock for part and disk. Useful for coordination of /// distributed operations which can lead to data duplication. Implemented only in ReplicatedMergeTree. virtual std::optional tryCreateZeroCopyExclusiveLock(const String &, const DiskPtr &) { return std::nullopt; } + virtual bool waitZeroCopyLockToDisappear(const ZeroCopyLock &, size_t) { return false; } /// Remove parts from disk calling part->remove(). Can do it in parallel in case of big set of parts and enabled settings. /// If we fail to remove some part and throw_on_error equal to `true` will throw an exception on the first failed part. diff --git a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index 0d123623f05..cf009a10c27 100644 --- a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -31,7 +31,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/src/Storages/MergeTree/MergeTreeDataPartChecksum.cpp b/src/Storages/MergeTree/MergeTreeDataPartChecksum.cpp index 8f4d066baa3..78f68ea72fe 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartChecksum.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartChecksum.cpp @@ -1,6 +1,6 @@ #include "MergeTreeDataPartChecksum.h" #include -#include +#include #include #include #include @@ -46,6 +46,10 @@ void MergeTreeDataPartChecksum::checkEqual(const MergeTreeDataPartChecksum & rhs void MergeTreeDataPartChecksum::checkSize(const IDataPartStorage & storage, const String & name) const { + /// Skip inverted index files, these have a default MergeTreeDataPartChecksum with file_size == 0 + if (name.ends_with(".gin_dict") || name.ends_with(".gin_post") || name.ends_with(".gin_seg") || name.ends_with(".gin_sid")) + return; + if (!storage.exists(name)) throw Exception(ErrorCodes::FILE_DOESNT_EXIST, "{} doesn't exist", fs::path(storage.getRelativePath()) / name); diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index db358e1b6a9..07da66e4378 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -588,18 +588,24 @@ MergeTreeDataSelectSamplingData MergeTreeDataSelectExecutor::getSampling( * It is also important that the entire universe can be covered using SAMPLE 0.1 OFFSET 0, ... OFFSET 0.9 and similar decimals. */ + auto parallel_replicas_mode = context->getParallelReplicasMode(); /// Parallel replicas has been requested but there is no way to sample data. /// Select all data from first replica and no data from other replicas. - if (settings.parallel_replicas_count > 1 && !data.supportsSampling() && settings.parallel_replica_offset > 0) + if (settings.parallel_replicas_count > 1 && parallel_replicas_mode == Context::ParallelReplicasMode::SAMPLE_KEY + && !data.supportsSampling() && settings.parallel_replica_offset > 0) { - LOG_DEBUG(log, "Will use no data on this replica because parallel replicas processing has been requested" + LOG_DEBUG( + log, + "Will use no data on this replica because parallel replicas processing has been requested" " (the setting 'max_parallel_replicas') but the table does not support sampling and this replica is not the first."); sampling.read_nothing = true; return sampling; } - sampling.use_sampling = relative_sample_size > 0 || (settings.parallel_replicas_count > 1 && data.supportsSampling()); - bool no_data = false; /// There is nothing left after sampling. + sampling.use_sampling = relative_sample_size > 0 + || (settings.parallel_replicas_count > 1 && parallel_replicas_mode == Context::ParallelReplicasMode::SAMPLE_KEY + && data.supportsSampling()); + bool no_data = false; /// There is nothing left after sampling. if (sampling.use_sampling) { diff --git a/src/Storages/MergeTree/MergeTreeIndexFullText.cpp b/src/Storages/MergeTree/MergeTreeIndexFullText.cpp index 80c4c201c3f..fa1bd36f863 100644 --- a/src/Storages/MergeTree/MergeTreeIndexFullText.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexFullText.cpp @@ -748,9 +748,7 @@ void bloomFilterIndexValidator(const IndexDescription & index, bool /*attach*/) if (!data_type.isString() && !data_type.isFixedString()) throw Exception(ErrorCodes::INCORRECT_QUERY, - "Bloom filter index can be used only with `String`, `FixedString`, " - "`LowCardinality(String)`, `LowCardinality(FixedString)` column " - "or Array with `String` or `FixedString` values column."); + "Ngram and token bloom filter indexes can only be used with column types `String`, `FixedString`, `LowCardinality(String)`, `LowCardinality(FixedString)`, `Array(String)` or `Array(FixedString)`"); } if (index.type == NgramTokenExtractor::getName()) diff --git a/src/Storages/MergeTree/MergeTreeIndexInverted.cpp b/src/Storages/MergeTree/MergeTreeIndexInverted.cpp index e7d86f2a635..8e8409f3868 100644 --- a/src/Storages/MergeTree/MergeTreeIndexInverted.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexInverted.cpp @@ -201,6 +201,7 @@ MergeTreeConditionInverted::MergeTreeConditionInverted( rpn.push_back(RPNElement::FUNCTION_UNKNOWN); return; } + rpn = std::move( RPNBuilder( query_info.filter_actions_dag->getOutputs().at(0), context_, @@ -208,10 +209,10 @@ MergeTreeConditionInverted::MergeTreeConditionInverted( { return this->traverseAtomAST(node, out); }).extractRPN()); + return; } ASTPtr filter_node = buildFilterNode(query_info.query); - if (!filter_node) { rpn.push_back(RPNElement::FUNCTION_UNKNOWN); @@ -226,7 +227,6 @@ MergeTreeConditionInverted::MergeTreeConditionInverted( query_info.prepared_sets, [&](const RPNBuilderTreeNode & node, RPNElement & out) { return traverseAtomAST(node, out); }); rpn = std::move(builder).extractRPN(); - } /// Keep in-sync with MergeTreeConditionFullText::alwaysUnknownOrTrue diff --git a/src/Storages/MergeTree/MergeTreeIndices.cpp b/src/Storages/MergeTree/MergeTreeIndices.cpp index 2be9ecd8de3..6ae96d00171 100644 --- a/src/Storages/MergeTree/MergeTreeIndices.cpp +++ b/src/Storages/MergeTree/MergeTreeIndices.cpp @@ -35,6 +35,7 @@ MergeTreeIndexPtr MergeTreeIndexFactory::get( { auto it = creators.find(index.type); if (it == creators.end()) + { throw Exception(ErrorCodes::INCORRECT_QUERY, "Unknown Index type '{}'. Available index types: {}", index.type, std::accumulate(creators.cbegin(), creators.cend(), std::string{}, @@ -46,6 +47,7 @@ MergeTreeIndexPtr MergeTreeIndexFactory::get( return left + ", " + right.first; }) ); + } return it->second(index); } @@ -61,8 +63,31 @@ MergeTreeIndices MergeTreeIndexFactory::getMany(const std::vectorhasArrayJoin()) + throw Exception(ErrorCodes::INCORRECT_QUERY, "Secondary index '{}' cannot contain array joins", index.name); + + try + { + index.expression->assertDeterministic(); + } + catch (Exception & e) + { + e.addMessage(fmt::format("for secondary index '{}'", index.name)); + throw; + } + + for (const auto & elem : index.sample_block) + if (elem.column && (isColumnConst(*elem.column) || elem.column->isDummy())) + throw Exception(ErrorCodes::INCORRECT_QUERY, "Secondary index '{}' cannot contain constants", index.name); + } + auto it = validators.find(index.type); if (it == validators.end()) + { throw Exception(ErrorCodes::INCORRECT_QUERY, "Unknown Index type '{}'. Available index types: {}", index.type, std::accumulate( @@ -77,6 +102,7 @@ void MergeTreeIndexFactory::validate(const IndexDescription & index, bool attach return left + ", " + right.first; }) ); + } it->second(index, attach); } diff --git a/src/Storages/MergeTree/MergeTreeMarksLoader.cpp b/src/Storages/MergeTree/MergeTreeMarksLoader.cpp index 3fc7ff54c35..ed8866b0044 100644 --- a/src/Storages/MergeTree/MergeTreeMarksLoader.cpp +++ b/src/Storages/MergeTree/MergeTreeMarksLoader.cpp @@ -1,13 +1,13 @@ -#include -#include -#include +#include #include #include -#include -#include -#include +#include +#include #include +#include #include +#include +#include #include @@ -15,6 +15,8 @@ namespace ProfileEvents { extern const Event WaitMarksLoadMicroseconds; extern const Event BackgroundLoadingMarksTasks; + extern const Event LoadedMarksCount; + extern const Event LoadedMarksMemoryBytes; } namespace DB @@ -62,7 +64,7 @@ MergeTreeMarksLoader::~MergeTreeMarksLoader() } -const MarkInCompressedFile & MergeTreeMarksLoader::getMark(size_t row_index, size_t column_index) +MarkInCompressedFile MergeTreeMarksLoader::getMark(size_t row_index, size_t column_index) { if (!marks) { @@ -87,7 +89,7 @@ const MarkInCompressedFile & MergeTreeMarksLoader::getMark(size_t row_index, siz throw Exception(ErrorCodes::LOGICAL_ERROR, "Column index: {} is out of range [0, {})", column_index, columns_in_mark); #endif - return (*marks)[row_index * columns_in_mark + column_index]; + return marks->get(row_index * columns_in_mark + column_index); } @@ -100,14 +102,17 @@ MarkCache::MappedPtr MergeTreeMarksLoader::loadMarksImpl() size_t mark_size = index_granularity_info.getMarkSizeInBytes(columns_in_mark); size_t expected_uncompressed_size = mark_size * marks_count; - auto res = std::make_shared(marks_count * columns_in_mark); + // We first read the marks into a temporary simple array, then compress them into a more compact + // representation. + PODArray plain_marks(marks_count * columns_in_mark); // temporary if (!index_granularity_info.mark_type.compressed && expected_uncompressed_size != file_size) throw Exception( ErrorCodes::CORRUPTED_DATA, "Bad size of marks file '{}': {}, must be: {}", std::string(fs::path(data_part_storage->getFullPath()) / mrk_path), - file_size, expected_uncompressed_size); + file_size, + expected_uncompressed_size); auto buffer = data_part_storage->readFile(mrk_path, read_settings.adjustBufferSize(file_size), file_size, std::nullopt); std::unique_ptr reader; @@ -119,12 +124,16 @@ MarkCache::MappedPtr MergeTreeMarksLoader::loadMarksImpl() if (!index_granularity_info.mark_type.adaptive) { /// Read directly to marks. - reader->readStrict(reinterpret_cast(res->data()), expected_uncompressed_size); + reader->readStrict(reinterpret_cast(plain_marks.data()), expected_uncompressed_size); if (!reader->eof()) - throw Exception(ErrorCodes::CANNOT_READ_ALL_DATA, + throw Exception( + ErrorCodes::CANNOT_READ_ALL_DATA, "Cannot read all marks from file {}, is eof: {}, buffer size: {}, file size: {}", - mrk_path, reader->eof(), reader->buffer().size(), file_size); + mrk_path, + reader->eof(), + reader->buffer().size(), + file_size); } else { @@ -132,7 +141,8 @@ MarkCache::MappedPtr MergeTreeMarksLoader::loadMarksImpl() size_t granularity; while (!reader->eof()) { - res->read(*reader, i * columns_in_mark, columns_in_mark); + reader->readStrict( + reinterpret_cast(plain_marks.data() + i * columns_in_mark), columns_in_mark * sizeof(MarkInCompressedFile)); readIntBinary(granularity, *reader); ++i; } @@ -141,7 +151,11 @@ MarkCache::MappedPtr MergeTreeMarksLoader::loadMarksImpl() throw Exception(ErrorCodes::CANNOT_READ_ALL_DATA, "Cannot read all marks from file {}", mrk_path); } - res->protect(); + auto res = std::make_shared(plain_marks); + + ProfileEvents::increment(ProfileEvents::LoadedMarksCount, marks_count * columns_in_mark); + ProfileEvents::increment(ProfileEvents::LoadedMarksMemoryBytes, res->approximateMemoryUsage()); + return res; } @@ -154,7 +168,7 @@ MarkCache::MappedPtr MergeTreeMarksLoader::loadMarks() auto key = mark_cache->hash(fs::path(data_part_storage->getFullPath()) / mrk_path); if (save_marks_in_cache) { - auto callback = [this]{ return loadMarksImpl(); }; + auto callback = [this] { return loadMarksImpl(); }; loaded_marks = mark_cache->getOrSet(key, callback); } else @@ -170,8 +184,7 @@ MarkCache::MappedPtr MergeTreeMarksLoader::loadMarks() if (!loaded_marks) { throw Exception( - ErrorCodes::LOGICAL_ERROR, "Failed to load marks: {}", - (fs::path(data_part_storage->getFullPath()) / mrk_path).string()); + ErrorCodes::LOGICAL_ERROR, "Failed to load marks: {}", (fs::path(data_part_storage->getFullPath()) / mrk_path).string()); } return loaded_marks; @@ -179,11 +192,14 @@ MarkCache::MappedPtr MergeTreeMarksLoader::loadMarks() std::future MergeTreeMarksLoader::loadMarksAsync() { - return scheduleFromThreadPool([this]() -> MarkCache::MappedPtr - { - ProfileEvents::increment(ProfileEvents::BackgroundLoadingMarksTasks); - return loadMarks(); - }, *load_marks_threadpool, "LoadMarksThread"); + return scheduleFromThreadPool( + [this]() -> MarkCache::MappedPtr + { + ProfileEvents::increment(ProfileEvents::BackgroundLoadingMarksTasks); + return loadMarks(); + }, + *load_marks_threadpool, + "LoadMarksThread"); } } diff --git a/src/Storages/MergeTree/MergeTreeMarksLoader.h b/src/Storages/MergeTree/MergeTreeMarksLoader.h index 4497339d767..0294cbbf3fc 100644 --- a/src/Storages/MergeTree/MergeTreeMarksLoader.h +++ b/src/Storages/MergeTree/MergeTreeMarksLoader.h @@ -30,7 +30,7 @@ public: ~MergeTreeMarksLoader(); - const MarkInCompressedFile & getMark(size_t row_index, size_t column_index = 0); + MarkInCompressedFile getMark(size_t row_index, size_t column_index = 0); private: DataPartStoragePtr data_part_storage; diff --git a/src/Storages/MergeTree/MergeTreePartition.cpp b/src/Storages/MergeTree/MergeTreePartition.cpp index 3a6908ef32d..3b28012e7d6 100644 --- a/src/Storages/MergeTree/MergeTreePartition.cpp +++ b/src/Storages/MergeTree/MergeTreePartition.cpp @@ -12,7 +12,7 @@ #include #include #include -#include +#include #include diff --git a/src/Storages/MergeTree/MergeTreeSettings.cpp b/src/Storages/MergeTree/MergeTreeSettings.cpp index e951b8f54cf..479e50fdebb 100644 --- a/src/Storages/MergeTree/MergeTreeSettings.cpp +++ b/src/Storages/MergeTree/MergeTreeSettings.cpp @@ -64,8 +64,7 @@ void MergeTreeSettings::loadFromQuery(ASTStorage & storage_def, ContextPtr conte auto ast = dynamic_cast(custom.getImpl()).ast; if (ast && isDiskFunction(ast)) { - const auto & ast_function = assert_cast(*ast); - auto disk_name = getOrCreateDiskFromDiskAST(ast_function, context); + auto disk_name = getOrCreateDiskFromDiskAST(ast, context); LOG_TRACE(&Poco::Logger::get("MergeTreeSettings"), "Created custom disk {}", disk_name); value = disk_name; } diff --git a/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp b/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp index 3a866cc8934..fdddc29048b 100644 --- a/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp +++ b/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp @@ -32,10 +32,12 @@ MergeTreeWhereOptimizer::MergeTreeWhereOptimizer( std::unordered_map column_sizes_, const StorageMetadataPtr & metadata_snapshot, const Names & queried_columns_, + const std::optional & supported_columns_, Poco::Logger * log_) : table_columns{collections::map( metadata_snapshot->getColumns().getAllPhysical(), [](const NameAndTypePair & col) { return col.name; })} , queried_columns{queried_columns_} + , supported_columns{supported_columns_} , sorting_key_names{NameSet( metadata_snapshot->getSortingKey().column_names.begin(), metadata_snapshot->getSortingKey().column_names.end())} , block_with_constants{KeyCondition::getBlockWithConstants(query_info.query->clone(), query_info.syntax_analyzer_result, context)} @@ -195,6 +197,8 @@ void MergeTreeWhereOptimizer::analyzeImpl(Conditions & res, const ASTPtr & node, && (!is_final || isExpressionOverSortingKey(node)) /// Only table columns are considered. Not array joined columns. NOTE We're assuming that aliases was expanded. && isSubsetOfTableColumns(cond.identifiers) + /// Some identifiers can unable to support PREWHERE (usually because of different types in Merge engine) + && identifiersSupportsPrewhere(cond.identifiers) /// Do not move conditions involving all queried columns. && cond.identifiers.size() < queried_columns.size(); @@ -321,6 +325,18 @@ UInt64 MergeTreeWhereOptimizer::getIdentifiersColumnSize(const NameSet & identif return size; } +bool MergeTreeWhereOptimizer::identifiersSupportsPrewhere(const NameSet & identifiers) const +{ + if (!supported_columns.has_value()) + return true; + + for (const auto & identifier : identifiers) + if (!supported_columns->contains(identifier)) + return false; + + return true; +} + bool MergeTreeWhereOptimizer::isExpressionOverSortingKey(const ASTPtr & ast) const { if (const auto * func = ast->as()) diff --git a/src/Storages/MergeTree/MergeTreeWhereOptimizer.h b/src/Storages/MergeTree/MergeTreeWhereOptimizer.h index f37255bdbee..8953923542e 100644 --- a/src/Storages/MergeTree/MergeTreeWhereOptimizer.h +++ b/src/Storages/MergeTree/MergeTreeWhereOptimizer.h @@ -39,6 +39,7 @@ public: std::unordered_map column_sizes_, const StorageMetadataPtr & metadata_snapshot, const Names & queried_columns_, + const std::optional & supported_columns_, Poco::Logger * log_); private: @@ -82,6 +83,7 @@ private: void optimizeArbitrary(ASTSelectQuery & select) const; UInt64 getIdentifiersColumnSize(const NameSet & identifiers) const; + bool identifiersSupportsPrewhere(const NameSet & identifiers) const; bool isExpressionOverSortingKey(const ASTPtr & ast) const; @@ -105,6 +107,7 @@ private: const StringSet table_columns; const Names queried_columns; + const std::optional supported_columns; const NameSet sorting_key_names; const Block block_with_constants; Poco::Logger * log; diff --git a/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp b/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp index 03829f1daf9..f6fc40884a1 100644 --- a/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp +++ b/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp @@ -86,7 +86,6 @@ MergedColumnOnlyOutputStream::fillChecksums( } new_part->setColumns(columns, serialization_infos); - return checksums; } diff --git a/src/Storages/MergeTree/MutateFromLogEntryTask.cpp b/src/Storages/MergeTree/MutateFromLogEntryTask.cpp index 4428f6c2bce..46a875fbed9 100644 --- a/src/Storages/MergeTree/MutateFromLogEntryTask.cpp +++ b/src/Storages/MergeTree/MutateFromLogEntryTask.cpp @@ -127,7 +127,7 @@ ReplicatedMergeMutateTaskBase::PrepareResult MutateFromLogEntryTask::prepare() zero_copy_lock = storage.tryCreateZeroCopyExclusiveLock(entry.new_part_name, disk); - if (!zero_copy_lock) + if (!zero_copy_lock || !zero_copy_lock->isLocked()) { LOG_DEBUG(log, "Mutation of part {} started by some other replica, will wait it and mutated merged part", entry.new_part_name); return PrepareResult{ diff --git a/src/Storages/MergeTree/MutateTask.cpp b/src/Storages/MergeTree/MutateTask.cpp index bcb1d5d2c28..9f7a12745c6 100644 --- a/src/Storages/MergeTree/MutateTask.cpp +++ b/src/Storages/MergeTree/MutateTask.cpp @@ -19,6 +19,7 @@ #include #include #include +#include #include #include @@ -204,20 +205,44 @@ getColumnsForNewDataPart( } SerializationInfoByName new_serialization_infos; - for (const auto & [name, info] : serialization_infos) + for (const auto & [name, old_info] : serialization_infos) { if (removed_columns.contains(name)) continue; auto it = renamed_columns_from_to.find(name); - if (it != renamed_columns_from_to.end()) - new_serialization_infos.emplace(it->second, info); - else - new_serialization_infos.emplace(name, info); + auto new_name = it == renamed_columns_from_to.end() ? name : it->second; + + if (!updated_header.has(new_name)) + { + new_serialization_infos.emplace(new_name, old_info); + continue; + } + + auto old_type = part_columns.getPhysical(name).type; + auto new_type = updated_header.getByName(new_name).type; + + SerializationInfo::Settings settings + { + .ratio_of_defaults_for_sparse = source_part->storage.getSettings()->ratio_of_defaults_for_sparse_serialization, + .choose_kind = false + }; + + if (!new_type->supportsSparseSerialization() || settings.isAlwaysDefault()) + continue; + + auto new_info = new_type->createSerializationInfo(settings); + if (!old_info->structureEquals(*new_info)) + { + new_serialization_infos.emplace(new_name, std::move(new_info)); + continue; + } + + new_info = old_info->createWithType(*old_type, *new_type, settings); + new_serialization_infos.emplace(new_name, std::move(new_info)); } - /// In compact parts we read all columns, because they all stored in a - /// single file + /// In compact parts we read all columns, because they all stored in a single file if (!isWidePart(source_part) || !isFullPartStorage(source_part->getDataPartStorage())) return {updated_header.getNamesAndTypesList(), new_serialization_infos}; @@ -626,6 +651,12 @@ static NameToNameVector collectFilesForRenames( } } + if (!source_part->getSerializationInfos().empty() + && new_part->getSerializationInfos().empty()) + { + rename_vector.emplace_back(IMergeTreeDataPart::SERIALIZATION_FILE_NAME, ""); + } + return rename_vector; } @@ -1224,8 +1255,8 @@ private: skip_part_indices, ctx->compression_codec, ctx->txn, - false, - false, + /*reset_columns=*/ true, + /*blocks_are_granules_size=*/ false, ctx->context->getWriteSettings()); ctx->mutating_pipeline = QueryPipelineBuilder::getPipeline(std::move(builder)); @@ -1542,6 +1573,45 @@ bool MutateTask::execute() return false; } +static bool canSkipConversionToNullable(const MergeTreeDataPartPtr & part, const MutationCommand & command) +{ + if (command.type != MutationCommand::READ_COLUMN) + return false; + + auto part_column = part->tryGetColumn(command.column_name); + if (!part_column) + return false; + + /// For ALTER MODIFY COLUMN from 'Type' to 'Nullable(Type)' we can skip mutatation and + /// apply only metadata conversion. But it doesn't work for custom serialization. + const auto * to_nullable = typeid_cast(command.data_type.get()); + if (!to_nullable) + return false; + + if (!part_column->type->equals(*to_nullable->getNestedType())) + return false; + + auto serialization = part->getSerialization(command.column_name); + if (serialization->getKind() != ISerialization::Kind::DEFAULT) + return false; + + return true; +} + +static bool canSkipMutationCommandForPart(const MergeTreeDataPartPtr & part, const MutationCommand & command, const ContextPtr & context) +{ + if (command.partition) + { + auto command_partition_id = part->storage.getPartitionIDFromQuery(command.partition, context); + if (part->info.partition_id != command_partition_id) + return true; + } + + if (canSkipConversionToNullable(part, command)) + return true; + + return false; +} bool MutateTask::prepare() { @@ -1560,11 +1630,8 @@ bool MutateTask::prepare() context_for_reading->setSetting("force_primary_key", false); for (const auto & command : *ctx->commands) - { - if (command.partition == nullptr || ctx->source_part->info.partition_id == ctx->data->getPartitionIDFromQuery( - command.partition, context_for_reading)) + if (!canSkipMutationCommandForPart(ctx->source_part, command, context_for_reading)) ctx->commands_for_part.emplace_back(command); - } if (ctx->source_part->isStoredOnDisk() && !isStorageTouchedByMutations( *ctx->data, ctx->source_part, ctx->metadata_snapshot, ctx->commands_for_part, context_for_reading)) diff --git a/src/Storages/MergeTree/PartMetadataManagerWithCache.cpp b/src/Storages/MergeTree/PartMetadataManagerWithCache.cpp index b6260d5edb6..3a53cf25745 100644 --- a/src/Storages/MergeTree/PartMetadataManagerWithCache.cpp +++ b/src/Storages/MergeTree/PartMetadataManagerWithCache.cpp @@ -1,7 +1,7 @@ #include "PartMetadataManagerWithCache.h" #if USE_ROCKSDB -#include +#include #include #include #include diff --git a/src/Storages/MergeTree/RPNBuilder.cpp b/src/Storages/MergeTree/RPNBuilder.cpp index cee5038ed21..fb3592a1541 100644 --- a/src/Storages/MergeTree/RPNBuilder.cpp +++ b/src/Storages/MergeTree/RPNBuilder.cpp @@ -59,7 +59,7 @@ void appendColumnNameWithoutAlias(const ActionsDAG::Node & node, WriteBuffer & o { auto name = node.function_base->getName(); if (legacy && name == "modulo") - writeCString("moduleLegacy", out); + writeCString("moduloLegacy", out); else writeString(name, out); diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeAttachThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeAttachThread.cpp index 557123ddae2..c859c994818 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeAttachThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeAttachThread.cpp @@ -60,11 +60,11 @@ void ReplicatedMergeTreeAttachThread::run() if (needs_retry) { - LOG_ERROR(log, "Initialization failed. Error: {}", e.message()); + LOG_ERROR(log, "Initialization failed. Error: {}", getCurrentExceptionMessage(/* with_stacktrace */ true)); } else { - LOG_ERROR(log, "Initialization failed, table will remain readonly. Error: {}", e.message()); + LOG_ERROR(log, "Initialization failed, table will remain readonly. Error: {}", getCurrentExceptionMessage(/* with_stacktrace */ true)); storage.initialization_done = true; } } diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index 20dbaa73812..9e6090c947b 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -1231,8 +1231,7 @@ bool ReplicatedMergeTreeQueue::isCoveredByFuturePartsImpl(const LogEntry & entry "because it is not disjoint with part {} that is currently executing."; /// This message can be too noisy, do not print it more than once per second - if (!(entry.last_postpone_time == time(nullptr) && entry.postpone_reason.ends_with("that is currently executing."))) - LOG_TEST(LogToStr(out_reason, log), fmt_string, entry.znode_name, new_part_name, future_part_elem.first); + LOG_TEST(LogToStr(out_reason, LogFrequencyLimiter(log, 5)), fmt_string, entry.znode_name, new_part_name, future_part_elem.first); return true; } @@ -1423,7 +1422,7 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry( { constexpr auto fmt_string = "Not executing log entry {} of type {} for part {}" " because source parts size ({}) is greater than the current maximum ({})."; - LOG_DEBUG(LogToStr(out_postpone_reason, log), fmt_string, entry.znode_name, entry.typeToString(), entry.new_part_name, + LOG_DEBUG(LogToStr(out_postpone_reason, LogFrequencyLimiter(log, 5)), fmt_string, entry.znode_name, entry.typeToString(), entry.new_part_name, ReadableSize(sum_parts_size_in_bytes), ReadableSize(max_source_parts_size)); return false; diff --git a/src/Storages/MergeTree/SimpleMergeSelector.cpp b/src/Storages/MergeTree/SimpleMergeSelector.cpp index 15291622a2a..af3373fd175 100644 --- a/src/Storages/MergeTree/SimpleMergeSelector.cpp +++ b/src/Storages/MergeTree/SimpleMergeSelector.cpp @@ -1,6 +1,6 @@ #include -#include +#include #include #include @@ -28,7 +28,7 @@ struct Estimator { double difference = std::abs(log2(static_cast(sum_size) / size_prev_at_left)); if (difference < settings.heuristic_to_align_parts_max_absolute_difference_in_powers_of_two) - current_score *= interpolateLinear(settings.heuristic_to_align_parts_max_score_adjustment, 1, + current_score *= std::lerp(settings.heuristic_to_align_parts_max_score_adjustment, 1, difference / settings.heuristic_to_align_parts_max_absolute_difference_in_powers_of_two); } @@ -115,8 +115,8 @@ bool allow( // std::cerr << "size_normalized: " << size_normalized << "\n"; /// Calculate boundaries for age - double min_age_to_lower_base = interpolateLinear(settings.min_age_to_lower_base_at_min_size, settings.min_age_to_lower_base_at_max_size, size_normalized); - double max_age_to_lower_base = interpolateLinear(settings.max_age_to_lower_base_at_min_size, settings.max_age_to_lower_base_at_max_size, size_normalized); + double min_age_to_lower_base = std::lerp(settings.min_age_to_lower_base_at_min_size, settings.min_age_to_lower_base_at_max_size, size_normalized); + double max_age_to_lower_base = std::lerp(settings.max_age_to_lower_base_at_min_size, settings.max_age_to_lower_base_at_max_size, size_normalized); // std::cerr << "min_age_to_lower_base: " << min_age_to_lower_base << "\n"; // std::cerr << "max_age_to_lower_base: " << max_age_to_lower_base << "\n"; @@ -137,7 +137,7 @@ bool allow( // std::cerr << "combined_ratio: " << combined_ratio << "\n"; - double lowered_base = interpolateLinear(settings.base, 2.0, combined_ratio); + double lowered_base = std::lerp(settings.base, 2.0, combined_ratio); // std::cerr << "------- lowered_base: " << lowered_base << "\n"; diff --git a/src/Storages/MergeTree/ZeroCopyLock.h b/src/Storages/MergeTree/ZeroCopyLock.h index 4e73b27804c..4400ea55b8f 100644 --- a/src/Storages/MergeTree/ZeroCopyLock.h +++ b/src/Storages/MergeTree/ZeroCopyLock.h @@ -14,6 +14,7 @@ struct ZeroCopyLock { ZeroCopyLock(const zkutil::ZooKeeperPtr & zookeeper, const std::string & lock_path, const std::string & lock_message); + bool isLocked() const { return lock->isLocked(); } /// Actual lock std::unique_ptr lock; }; diff --git a/src/Storages/MergeTree/tests/gtest_executor.cpp b/src/Storages/MergeTree/tests/gtest_executor.cpp index e45887da7ef..3a4f147b456 100644 --- a/src/Storages/MergeTree/tests/gtest_executor.cpp +++ b/src/Storages/MergeTree/tests/gtest_executor.cpp @@ -15,6 +15,7 @@ using namespace DB; namespace CurrentMetrics { extern const Metric BackgroundMergesAndMutationsPoolTask; + extern const Metric BackgroundMergesAndMutationsPoolSize; } std::random_device device; @@ -102,7 +103,8 @@ TEST(Executor, Simple) "GTest", 1, // threads 100, // max_tasks - CurrentMetrics::BackgroundMergesAndMutationsPoolTask + CurrentMetrics::BackgroundMergesAndMutationsPoolTask, + CurrentMetrics::BackgroundMergesAndMutationsPoolSize ); String schedule; // mutex is not required because we have a single worker @@ -144,7 +146,8 @@ TEST(Executor, RemoveTasks) "GTest", tasks_kinds, tasks_kinds * batch, - CurrentMetrics::BackgroundMergesAndMutationsPoolTask + CurrentMetrics::BackgroundMergesAndMutationsPoolTask, + CurrentMetrics::BackgroundMergesAndMutationsPoolSize ); for (size_t i = 0; i < batch; ++i) @@ -184,7 +187,8 @@ TEST(Executor, RemoveTasksStress) "GTest", tasks_kinds, tasks_kinds * batch * (schedulers_count + removers_count), - CurrentMetrics::BackgroundMergesAndMutationsPoolTask + CurrentMetrics::BackgroundMergesAndMutationsPoolTask, + CurrentMetrics::BackgroundMergesAndMutationsPoolSize ); std::barrier barrier(schedulers_count + removers_count); @@ -234,7 +238,8 @@ TEST(Executor, UpdatePolicy) "GTest", 1, // threads 100, // max_tasks - CurrentMetrics::BackgroundMergesAndMutationsPoolTask + CurrentMetrics::BackgroundMergesAndMutationsPoolTask, + CurrentMetrics::BackgroundMergesAndMutationsPoolSize ); String schedule; // mutex is not required because we have a single worker diff --git a/src/Storages/PostgreSQL/MaterializedPostgreSQLConsumer.cpp b/src/Storages/PostgreSQL/MaterializedPostgreSQLConsumer.cpp index 9c6eeceb605..d048c94ac75 100644 --- a/src/Storages/PostgreSQL/MaterializedPostgreSQLConsumer.cpp +++ b/src/Storages/PostgreSQL/MaterializedPostgreSQLConsumer.cpp @@ -2,7 +2,7 @@ #include "StorageMaterializedPostgreSQL.h" #include -#include +#include #include #include #include diff --git a/src/Storages/SelectQueryInfo.h b/src/Storages/SelectQueryInfo.h index 40ea84ec68b..db43b2fc3f8 100644 --- a/src/Storages/SelectQueryInfo.h +++ b/src/Storages/SelectQueryInfo.h @@ -207,6 +207,8 @@ struct SelectQueryInfo /// /// Configured in StorageDistributed::getQueryProcessingStage() ClusterPtr optimized_cluster; + /// should we use custom key with the cluster + bool use_custom_key = false; mutable ParallelReplicasReadingCoordinatorPtr coordinator; @@ -218,6 +220,8 @@ struct SelectQueryInfo /// It is needed for PK analysis based on row_level_policy and additional_filters. ASTs filter_asts; + ASTPtr parallel_replica_custom_key_ast; + /// Filter actions dag for current storage ActionsDAGPtr filter_actions_dag; @@ -250,6 +254,7 @@ struct SelectQueryInfo MergeTreeDataSelectAnalysisResultPtr merge_tree_select_result_ptr; bool is_parameterized_view = false; + NameToNameMap parameterized_view_values; // If limit is not 0, that means it's a trivial limit query. UInt64 limit = 0; diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index 5c0027b20f5..0be4ae3a79f 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -18,6 +18,7 @@ #include #include #include +#include #include @@ -39,11 +40,16 @@ #include #include +#include +#include #include #include #include +#include +#include #include #include +#include #include #include @@ -51,9 +57,11 @@ #include #include #include +#include #include #include #include +#include #include #include #include @@ -65,13 +73,17 @@ #include #include #include +#include + #include +#include #include #include #include #include +#include #include #include #include @@ -82,6 +94,7 @@ #include #include +#include #include #include @@ -134,6 +147,7 @@ namespace ErrorCodes extern const int DISTRIBUTED_TOO_MANY_PENDING_BYTES; extern const int ARGUMENT_OUT_OF_BOUND; extern const int TOO_LARGE_DISTRIBUTED_DEPTH; + extern const int DISTRIBUTED_IN_JOIN_SUBQUERY_DENIED; } namespace ActionLocks @@ -262,7 +276,6 @@ size_t getClusterQueriedNodes(const Settings & settings, const ClusterPtr & clus } - /// For destruction of std::unique_ptr of type that is incomplete in class definition. StorageDistributed::~StorageDistributed() = default; @@ -400,29 +413,38 @@ QueryProcessingStage::Enum StorageDistributed::getQueryProcessingStage( const auto & settings = local_context->getSettingsRef(); ClusterPtr cluster = getCluster(); - query_info.cluster = cluster; size_t nodes = getClusterQueriedNodes(settings, cluster); - /// Always calculate optimized cluster here, to avoid conditions during read() - /// (Anyway it will be calculated in the read()) - if (nodes > 1 && settings.optimize_skip_unused_shards) + if (query_info.use_custom_key) { - ClusterPtr optimized_cluster = getOptimizedCluster(local_context, storage_snapshot, query_info.query); - if (optimized_cluster) - { - LOG_DEBUG(log, "Skipping irrelevant shards - the query will be sent to the following shards of the cluster (shard numbers): {}", - makeFormattedListOfShards(optimized_cluster)); + LOG_INFO(log, "Single shard cluster used with custom_key, transforming replicas into virtual shards"); + query_info.cluster = cluster->getClusterWithReplicasAsShards(settings, settings.max_parallel_replicas); + } + else + { + query_info.cluster = cluster; - cluster = optimized_cluster; - query_info.optimized_cluster = cluster; - - nodes = getClusterQueriedNodes(settings, cluster); - } - else + if (nodes > 1 && settings.optimize_skip_unused_shards) { - LOG_DEBUG(log, "Unable to figure out irrelevant shards from WHERE/PREWHERE clauses - the query will be sent to all shards of the cluster{}", - has_sharding_key ? "" : " (no sharding key)"); + /// Always calculate optimized cluster here, to avoid conditions during read() + /// (Anyway it will be calculated in the read()) + ClusterPtr optimized_cluster = getOptimizedCluster(local_context, storage_snapshot, query_info.query); + if (optimized_cluster) + { + LOG_DEBUG(log, "Skipping irrelevant shards - the query will be sent to the following shards of the cluster (shard numbers): {}", + makeFormattedListOfShards(optimized_cluster)); + + cluster = optimized_cluster; + query_info.optimized_cluster = cluster; + + nodes = getClusterQueriedNodes(settings, cluster); + } + else + { + LOG_DEBUG(log, "Unable to figure out irrelevant shards from WHERE/PREWHERE clauses - the query will be sent to all shards of the cluster{}", + has_sharding_key ? "" : " (no sharding key)"); + } } } @@ -622,12 +644,278 @@ StorageSnapshotPtr StorageDistributed::getStorageSnapshotForQuery( namespace { -QueryTreeNodePtr buildQueryTreeDistributedTableReplacedWithLocalTable(const SelectQueryInfo & query_info, +/// Visitor that collect column source to columns mapping from query and all subqueries +class CollectColumnSourceToColumnsVisitor : public InDepthQueryTreeVisitor +{ +public: + struct Columns + { + NameSet column_names; + NamesAndTypes columns; + + void addColumn(NameAndTypePair column) + { + if (column_names.contains(column.name)) + return; + + column_names.insert(column.name); + columns.push_back(std::move(column)); + } + }; + + const std::unordered_map & getColumnSourceToColumns() const + { + return column_source_to_columns; + } + + void visitImpl(QueryTreeNodePtr & node) + { + auto * column_node = node->as(); + if (!column_node) + return; + + auto column_source = column_node->getColumnSourceOrNull(); + if (!column_source) + return; + + auto it = column_source_to_columns.find(column_source); + if (it == column_source_to_columns.end()) + { + auto [insert_it, _] = column_source_to_columns.emplace(column_source, Columns()); + it = insert_it; + } + + it->second.addColumn(column_node->getColumn()); + } + +private: + std::unordered_map column_source_to_columns; +}; + +/** Visitor that rewrites IN and JOINs in query and all subqueries according to distributed_product_mode and + * prefer_global_in_and_join settings. + * + * Additionally collects GLOBAL JOIN and GLOBAL IN query nodes. + * + * If distributed_product_mode = deny, then visitor throws exception if there are multiple distributed tables. + * If distributed_product_mode = local, then visitor collects replacement map for tables that must be replaced + * with local tables. + * If distributed_product_mode = global or prefer_global_in_and_join setting is true, then visitor rewrites JOINs and IN functions that + * contain distributed tables to GLOBAL JOINs and GLOBAL IN functions. + * If distributed_product_mode = allow, then visitor does not rewrite query if there are multiple distributed tables. + */ +class DistributedProductModeRewriteInJoinVisitor : public InDepthQueryTreeVisitorWithContext +{ +public: + using Base = InDepthQueryTreeVisitorWithContext; + using Base::Base; + + explicit DistributedProductModeRewriteInJoinVisitor(const ContextPtr & context_) + : Base(context_) + {} + + struct InFunctionOrJoin + { + QueryTreeNodePtr query_node; + size_t subquery_depth = 0; + }; + + const std::unordered_map & getReplacementMap() const + { + return replacement_map; + } + + const std::vector & getGlobalInOrJoinNodes() const + { + return global_in_or_join_nodes; + } + + static bool needChildVisit(QueryTreeNodePtr & parent, QueryTreeNodePtr & child) + { + auto * function_node = parent->as(); + if (function_node && isNameOfGlobalInFunction(function_node->getFunctionName())) + return false; + + auto * join_node = parent->as(); + if (join_node && join_node->getLocality() == JoinLocality::Global && join_node->getRightTableExpression() == child) + return false; + + return true; + } + + void visitImpl(QueryTreeNodePtr & node) + { + auto * function_node = node->as(); + auto * join_node = node->as(); + + if ((function_node && isNameOfGlobalInFunction(function_node->getFunctionName())) || + (join_node && join_node->getLocality() == JoinLocality::Global)) + { + InFunctionOrJoin in_function_or_join_entry; + in_function_or_join_entry.query_node = node; + in_function_or_join_entry.subquery_depth = getSubqueryDepth(); + global_in_or_join_nodes.push_back(std::move(in_function_or_join_entry)); + return; + } + + if ((function_node && isNameOfLocalInFunction(function_node->getFunctionName())) || + (join_node && join_node->getLocality() != JoinLocality::Global)) + { + InFunctionOrJoin in_function_or_join_entry; + in_function_or_join_entry.query_node = node; + in_function_or_join_entry.subquery_depth = getSubqueryDepth(); + in_function_or_join_stack.push_back(in_function_or_join_entry); + return; + } + + if (node->getNodeType() == QueryTreeNodeType::TABLE) + tryRewriteTableNodeIfNeeded(node); + } + + void leaveImpl(QueryTreeNodePtr & node) + { + if (!in_function_or_join_stack.empty() && node.get() == in_function_or_join_stack.back().query_node.get()) + in_function_or_join_stack.pop_back(); + } + +private: + void tryRewriteTableNodeIfNeeded(const QueryTreeNodePtr & table_node) + { + const auto & table_node_typed = table_node->as(); + const auto * distributed_storage = typeid_cast(table_node_typed.getStorage().get()); + if (!distributed_storage) + return; + + bool distributed_valid_for_rewrite = distributed_storage->getShardCount() >= 2; + if (!distributed_valid_for_rewrite) + return; + + auto distributed_product_mode = getSettings().distributed_product_mode; + + if (distributed_product_mode == DistributedProductMode::LOCAL) + { + StorageID remote_storage_id = StorageID{distributed_storage->getRemoteDatabaseName(), + distributed_storage->getRemoteTableName()}; + auto resolved_remote_storage_id = getContext()->resolveStorageID(remote_storage_id); + const auto & distributed_storage_columns = table_node_typed.getStorageSnapshot()->metadata->getColumns(); + auto storage = std::make_shared(resolved_remote_storage_id, distributed_storage_columns); + auto replacement_table_expression = std::make_shared(std::move(storage), getContext()); + replacement_map.emplace(table_node.get(), std::move(replacement_table_expression)); + } + else if ((distributed_product_mode == DistributedProductMode::GLOBAL || getSettings().prefer_global_in_and_join) && + !in_function_or_join_stack.empty()) + { + auto * in_or_join_node_to_modify = in_function_or_join_stack.back().query_node.get(); + + if (auto * in_function_to_modify = in_or_join_node_to_modify->as()) + { + auto global_in_function_name = getGlobalInFunctionNameForLocalInFunctionName(in_function_to_modify->getFunctionName()); + auto global_in_function_resolver = FunctionFactory::instance().get(global_in_function_name, getContext()); + in_function_to_modify->resolveAsFunction(global_in_function_resolver->build(in_function_to_modify->getArgumentColumns())); + } + else if (auto * join_node_to_modify = in_or_join_node_to_modify->as()) + { + join_node_to_modify->setLocality(JoinLocality::Global); + } + + global_in_or_join_nodes.push_back(in_function_or_join_stack.back()); + } + else if (distributed_product_mode == DistributedProductMode::ALLOW) + { + return; + } + else if (distributed_product_mode == DistributedProductMode::DENY) + { + throw Exception(ErrorCodes::DISTRIBUTED_IN_JOIN_SUBQUERY_DENIED, + "Double-distributed IN/JOIN subqueries is denied (distributed_product_mode = 'deny'). " + "You may rewrite query to use local tables " + "in subqueries, or use GLOBAL keyword, or set distributed_product_mode to suitable value."); + } + } + + std::vector in_function_or_join_stack; + std::unordered_map replacement_map; + std::vector global_in_or_join_nodes; +}; + +/** Execute subquery node and put result in mutable context temporary table. + * Returns table node that is initialized with temporary table storage. + */ +QueryTreeNodePtr executeSubqueryNode(const QueryTreeNodePtr & subquery_node, + ContextMutablePtr & mutable_context, + size_t subquery_depth) +{ + auto subquery_hash = subquery_node->getTreeHash(); + String temporary_table_name = fmt::format("_data_{}_{}", subquery_hash.first, subquery_hash.second); + + const auto & external_tables = mutable_context->getExternalTables(); + auto external_table_it = external_tables.find(temporary_table_name); + if (external_table_it != external_tables.end()) + { + auto temporary_table_expression_node = std::make_shared(external_table_it->second, mutable_context); + temporary_table_expression_node->setTemporaryTableName(temporary_table_name); + return temporary_table_expression_node; + } + + auto subquery_options = SelectQueryOptions(QueryProcessingStage::Complete, subquery_depth, true /*is_subquery*/); + auto context_copy = Context::createCopy(mutable_context); + updateContextForSubqueryExecution(context_copy); + + InterpreterSelectQueryAnalyzer interpreter(subquery_node, context_copy, subquery_options); + auto & query_plan = interpreter.getQueryPlan(); + + auto sample_block_with_unique_names = query_plan.getCurrentDataStream().header; + makeUniqueColumnNamesInBlock(sample_block_with_unique_names); + + if (!blocksHaveEqualStructure(sample_block_with_unique_names, query_plan.getCurrentDataStream().header)) + { + auto actions_dag = ActionsDAG::makeConvertingActions( + query_plan.getCurrentDataStream().header.getColumnsWithTypeAndName(), + sample_block_with_unique_names.getColumnsWithTypeAndName(), + ActionsDAG::MatchColumnsMode::Position); + auto converting_step = std::make_unique(query_plan.getCurrentDataStream(), std::move(actions_dag)); + query_plan.addStep(std::move(converting_step)); + } + + Block sample = interpreter.getSampleBlock(); + NamesAndTypesList columns = sample.getNamesAndTypesList(); + + auto external_storage_holder = TemporaryTableHolder( + mutable_context, + ColumnsDescription{columns}, + ConstraintsDescription{}, + nullptr /*query*/, + true /*create_for_global_subquery*/); + + StoragePtr external_storage = external_storage_holder.getTable(); + auto temporary_table_expression_node = std::make_shared(external_storage, mutable_context); + temporary_table_expression_node->setTemporaryTableName(temporary_table_name); + + auto table_out = external_storage->write({}, external_storage->getInMemoryMetadataPtr(), mutable_context); + auto io = interpreter.execute(); + io.pipeline.complete(std::move(table_out)); + CompletedPipelineExecutor executor(io.pipeline); + executor.execute(); + + mutable_context->addExternalTable(temporary_table_name, std::move(external_storage_holder)); + + return temporary_table_expression_node; +} + +QueryTreeNodePtr buildQueryTreeDistributed(SelectQueryInfo & query_info, const StorageSnapshotPtr & distributed_storage_snapshot, const StorageID & remote_storage_id, const ASTPtr & remote_table_function) { - const auto & query_context = query_info.planner_context->getQueryContext(); + auto & planner_context = query_info.planner_context; + const auto & query_context = planner_context->getQueryContext(); + + std::optional table_expression_modifiers; + + if (auto * query_info_table_node = query_info.table_expression->as()) + table_expression_modifiers = query_info_table_node->getTableExpressionModifiers(); + else if (auto * query_info_table_function_node = query_info.table_expression->as()) + table_expression_modifiers = query_info_table_function_node->getTableExpressionModifiers(); QueryTreeNodePtr replacement_table_expression; @@ -639,6 +927,9 @@ QueryTreeNodePtr buildQueryTreeDistributedTableReplacedWithLocalTable(const Sele auto table_function_node = std::make_shared(remote_table_function_node.getFunctionName()); table_function_node->getArgumentsNode() = remote_table_function_node.getArgumentsNode(); + if (table_expression_modifiers) + table_function_node->setTableExpressionModifiers(*table_expression_modifiers); + QueryAnalysisPass query_analysis_pass; query_analysis_pass.run(table_function_node, query_context); @@ -648,13 +939,91 @@ QueryTreeNodePtr buildQueryTreeDistributedTableReplacedWithLocalTable(const Sele { auto resolved_remote_storage_id = query_context->resolveStorageID(remote_storage_id); auto storage = std::make_shared(resolved_remote_storage_id, distributed_storage_snapshot->metadata->getColumns()); + auto table_node = std::make_shared(std::move(storage), query_context); - replacement_table_expression = std::make_shared(std::move(storage), query_context); + if (table_expression_modifiers) + table_node->setTableExpressionModifiers(*table_expression_modifiers); + + replacement_table_expression = std::move(table_node); } replacement_table_expression->setAlias(query_info.table_expression->getAlias()); - return query_info.query_tree->cloneAndReplace(query_info.table_expression, std::move(replacement_table_expression)); + auto query_tree_to_modify = query_info.query_tree->cloneAndReplace(query_info.table_expression, std::move(replacement_table_expression)); + + CollectColumnSourceToColumnsVisitor collect_column_source_to_columns_visitor; + collect_column_source_to_columns_visitor.visit(query_tree_to_modify); + + const auto & column_source_to_columns = collect_column_source_to_columns_visitor.getColumnSourceToColumns(); + + DistributedProductModeRewriteInJoinVisitor visitor(query_info.planner_context->getQueryContext()); + visitor.visit(query_tree_to_modify); + + auto replacement_map = visitor.getReplacementMap(); + const auto & global_in_or_join_nodes = visitor.getGlobalInOrJoinNodes(); + + for (const auto & global_in_or_join_node : global_in_or_join_nodes) + { + if (auto * join_node = global_in_or_join_node.query_node->as()) + { + auto join_right_table_expression = join_node->getRightTableExpression(); + auto join_right_table_expression_node_type = join_right_table_expression->getNodeType(); + + QueryTreeNodePtr subquery_node; + + if (join_right_table_expression_node_type == QueryTreeNodeType::QUERY || + join_right_table_expression_node_type == QueryTreeNodeType::UNION) + { + subquery_node = join_right_table_expression; + } + else if (join_right_table_expression_node_type == QueryTreeNodeType::TABLE || + join_right_table_expression_node_type == QueryTreeNodeType::TABLE_FUNCTION) + { + const auto & columns = column_source_to_columns.at(join_right_table_expression).columns; + subquery_node = buildSubqueryToReadColumnsFromTableExpression(columns, + join_right_table_expression, + planner_context->getQueryContext()); + } + else + { + throw Exception(ErrorCodes::LOGICAL_ERROR, + "Expected JOIN right table expression to be table, table function, query or union node. Actual {}", + join_right_table_expression->formatASTForErrorMessage()); + } + + auto temporary_table_expression_node = executeSubqueryNode(subquery_node, + planner_context->getMutableQueryContext(), + global_in_or_join_node.subquery_depth); + temporary_table_expression_node->setAlias(join_right_table_expression->getAlias()); + replacement_map.emplace(join_right_table_expression.get(), std::move(temporary_table_expression_node)); + continue; + } + else if (auto * in_function_node = global_in_or_join_node.query_node->as()) + { + auto & in_function_subquery_node = in_function_node->getArguments().getNodes().at(1); + auto in_function_node_type = in_function_subquery_node->getNodeType(); + if (in_function_node_type != QueryTreeNodeType::QUERY && in_function_node_type != QueryTreeNodeType::UNION) + continue; + + auto temporary_table_expression_node = executeSubqueryNode(in_function_subquery_node, + planner_context->getMutableQueryContext(), + global_in_or_join_node.subquery_depth); + in_function_subquery_node = std::move(temporary_table_expression_node); + } + else + { + throw Exception(ErrorCodes::LOGICAL_ERROR, + "Expected global IN or JOIN query node. Actual {}", + global_in_or_join_node.query_node->formatASTForErrorMessage()); + } + } + + if (!replacement_map.empty()) + query_tree_to_modify = query_tree_to_modify->cloneAndReplace(replacement_map); + + removeGroupingFunctionSpecializations(query_tree_to_modify); + + return query_tree_to_modify; } } @@ -682,17 +1051,13 @@ void StorageDistributed::read( if (!remote_table_function_ptr) remote_storage_id = StorageID{remote_database, remote_table}; - auto query_tree_with_replaced_distributed_table = buildQueryTreeDistributedTableReplacedWithLocalTable(query_info, + auto query_tree_distributed = buildQueryTreeDistributed(query_info, storage_snapshot, remote_storage_id, remote_table_function_ptr); - query_ast = queryNodeToSelectQuery(query_tree_with_replaced_distributed_table); - - Planner planner(query_tree_with_replaced_distributed_table, SelectQueryOptions(processed_stage).analyze()); - planner.buildQueryPlanIfNeeded(); - - header = planner.getQueryPlan().getCurrentDataStream().header; + query_ast = queryNodeToSelectQuery(query_tree_distributed); + header = InterpreterSelectQueryAnalyzer::getSampleBlock(query_ast, local_context, SelectQueryOptions(processed_stage).analyze()); } else { @@ -728,13 +1093,36 @@ void StorageDistributed::read( storage_snapshot, processed_stage); + auto settings = local_context->getSettingsRef(); + + ClusterProxy::AdditionalShardFilterGenerator additional_shard_filter_generator; + if (query_info.use_custom_key) + { + if (auto custom_key_ast = parseCustomKeyForTable(settings.parallel_replicas_custom_key, *local_context)) + { + if (query_info.getCluster()->getShardCount() == 1) + { + // we are reading from single shard with multiple replicas but didn't transform replicas + // into virtual shards with custom_key set + throw Exception(ErrorCodes::LOGICAL_ERROR, "Replicas weren't transformed into virtual shards"); + } + + additional_shard_filter_generator = + [&, custom_key_ast = std::move(custom_key_ast), shard_count = query_info.cluster->getShardCount()](uint64_t shard_num) -> ASTPtr + { + return getCustomKeyFilterForParallelReplica( + shard_count, shard_num - 1, custom_key_ast, settings.parallel_replicas_custom_key_filter_type, *this, local_context); + }; + } + } + ClusterProxy::executeQuery( query_plan, header, processed_stage, main_table, remote_table_function_ptr, select_stream_factory, log, modified_query_ast, local_context, query_info, sharding_key_expr, sharding_key_column_name, - query_info.cluster); + query_info.cluster, additional_shard_filter_generator); /// This is a bug, it is possible only when there is no shards to query, and this is handled earlier. if (!query_plan.isInitialized()) @@ -877,7 +1265,7 @@ std::optional StorageDistributed::distributedWriteBetweenDistribu /// INSERT SELECT query returns empty block auto remote_query_executor - = std::make_shared(shard_info.pool, std::move(connections), new_query_str, Block{}, query_context); + = std::make_shared(std::move(connections), new_query_str, Block{}, query_context); QueryPipeline remote_pipeline(std::make_shared(remote_query_executor, false, settings.async_socket_for_remote)); remote_pipeline.complete(std::make_shared(remote_query_executor->getHeader())); @@ -1054,7 +1442,7 @@ void StorageDistributed::initializeFromDisk() { pool.scheduleOrThrowOnError([&]() { - createDirectoryMonitors(disk); + initializeDirectoryQueuesForDisk(disk); }); } pool.wait(); @@ -1093,7 +1481,7 @@ void StorageDistributed::shutdown() void StorageDistributed::drop() { // Some INSERT in-between shutdown() and drop() can call - // requireDirectoryMonitor() again, so call shutdown() to clear them, but + // getDirectoryQueue() again, so call shutdown() to clear them, but // when the drop() (this function) executed none of INSERT is allowed in // parallel. // @@ -1156,7 +1544,7 @@ StoragePolicyPtr StorageDistributed::getStoragePolicy() const return storage_policy; } -void StorageDistributed::createDirectoryMonitors(const DiskPtr & disk) +void StorageDistributed::initializeDirectoryQueuesForDisk(const DiskPtr & disk) { const std::string path(disk->getPath() + relative_data_path); fs::create_directories(path); @@ -1168,12 +1556,15 @@ void StorageDistributed::createDirectoryMonitors(const DiskPtr & disk) const auto & dir_path = it->path(); if (std::filesystem::is_directory(dir_path)) { + /// Created by DistributedSink const auto & tmp_path = dir_path / "tmp"; - - /// "tmp" created by DistributedSink if (std::filesystem::is_directory(tmp_path) && std::filesystem::is_empty(tmp_path)) std::filesystem::remove(tmp_path); + const auto & broken_path = dir_path / "broken"; + if (std::filesystem::is_directory(broken_path) && std::filesystem::is_empty(broken_path)) + std::filesystem::remove(broken_path); + if (std::filesystem::is_empty(dir_path)) { LOG_DEBUG(log, "Removing {} (used for async INSERT into Distributed)", dir_path.string()); @@ -1182,14 +1573,14 @@ void StorageDistributed::createDirectoryMonitors(const DiskPtr & disk) } else { - requireDirectoryMonitor(disk, dir_path.filename().string()); + getDirectoryQueue(disk, dir_path.filename().string()); } } } } -StorageDistributedDirectoryMonitor& StorageDistributed::requireDirectoryMonitor(const DiskPtr & disk, const std::string & name) +DistributedAsyncInsertDirectoryQueue & StorageDistributed::getDirectoryQueue(const DiskPtr & disk, const std::string & name) { const std::string & disk_path = disk->getPath(); const std::string key(disk_path + name); @@ -1198,8 +1589,8 @@ StorageDistributedDirectoryMonitor& StorageDistributed::requireDirectoryMonitor( auto & node_data = cluster_nodes_data[key]; if (!node_data.directory_monitor) { - node_data.connection_pool = StorageDistributedDirectoryMonitor::createPool(name, *this); - node_data.directory_monitor = std::make_unique( + node_data.connection_pool = DistributedAsyncInsertDirectoryQueue::createPool(name, *this); + node_data.directory_monitor = std::make_unique( *this, disk, relative_data_path + name, node_data.connection_pool, monitors_blocker, @@ -1208,9 +1599,9 @@ StorageDistributedDirectoryMonitor& StorageDistributed::requireDirectoryMonitor( return *node_data.directory_monitor; } -std::vector StorageDistributed::getDirectoryMonitorsStatuses() const +std::vector StorageDistributed::getDirectoryQueueStatuses() const { - std::vector statuses; + std::vector statuses; std::lock_guard lock(cluster_nodes_mutex); statuses.reserve(cluster_nodes_data.size()); for (const auto & node : cluster_nodes_data) @@ -1221,7 +1612,7 @@ std::vector StorageDistributed::getD std::optional StorageDistributed::totalBytes(const Settings &) const { UInt64 total_bytes = 0; - for (const auto & status : getDirectoryMonitorsStatuses()) + for (const auto & status : getDirectoryQueueStatuses()) total_bytes += status.bytes_count; return total_bytes; } @@ -1382,7 +1773,7 @@ void StorageDistributed::flushClusterNodesAllData(ContextPtr local_context) /// Sync SYSTEM FLUSH DISTRIBUTED with TRUNCATE auto table_lock = lockForShare(local_context->getCurrentQueryId(), local_context->getSettingsRef().lock_acquire_timeout); - std::vector> directory_monitors; + std::vector> directory_monitors; { std::lock_guard lock(cluster_nodes_mutex); diff --git a/src/Storages/StorageDistributed.h b/src/Storages/StorageDistributed.h index 66fd7b77889..3a7fae44708 100644 --- a/src/Storages/StorageDistributed.h +++ b/src/Storages/StorageDistributed.h @@ -2,7 +2,7 @@ #include #include -#include +#include #include #include #include @@ -38,7 +38,8 @@ using ExpressionActionsPtr = std::shared_ptr; class StorageDistributed final : public IStorage, WithContext { friend class DistributedSink; - friend class StorageDistributedDirectoryMonitor; + friend class DistributedAsyncInsertBatch; + friend class DistributedAsyncInsertDirectoryQueue; friend class StorageSystemDistributionQueue; public: @@ -164,15 +165,19 @@ private: const String & getRelativeDataPath() const { return relative_data_path; } /// create directory monitors for each existing subdirectory - void createDirectoryMonitors(const DiskPtr & disk); - /// ensure directory monitor thread and connectoin pool creation by disk and subdirectory name - StorageDistributedDirectoryMonitor & requireDirectoryMonitor(const DiskPtr & disk, const std::string & name); + void initializeDirectoryQueuesForDisk(const DiskPtr & disk); + + /// Get directory queue thread and connection pool created by disk and subdirectory name + /// + /// Used for the INSERT into Distributed in case of insert_distributed_sync==1, from DistributedSink. + DistributedAsyncInsertDirectoryQueue & getDirectoryQueue(const DiskPtr & disk, const std::string & name); + /// Return list of metrics for all created monitors /// (note that monitors are created lazily, i.e. until at least one INSERT executed) /// /// Used by StorageSystemDistributionQueue - std::vector getDirectoryMonitorsStatuses() const; + std::vector getDirectoryQueueStatuses() const; static IColumn::Selector createSelector(ClusterPtr cluster, const ColumnWithTypeAndName & result); /// Apply the following settings: @@ -247,7 +252,7 @@ private: struct ClusterNodeData { - std::shared_ptr directory_monitor; + std::shared_ptr directory_monitor; ConnectionPoolPtr connection_pool; }; std::unordered_map cluster_nodes_data; diff --git a/src/Storages/StorageFile.cpp b/src/Storages/StorageFile.cpp index e2a2f84bc72..5fd5664b9e6 100644 --- a/src/Storages/StorageFile.cpp +++ b/src/Storages/StorageFile.cpp @@ -3,7 +3,7 @@ #include #include #include -#include +#include #include #include @@ -368,8 +368,7 @@ ColumnsDescription StorageFile::getTableStructureFromFile( if (paths.empty()) throw Exception(ErrorCodes::INCORRECT_FILE_NAME, "Cannot get table structure from file, because no files match specified name"); - auto source = StorageDistributedDirectoryMonitor::createSourceFromFile(paths[0]); - return ColumnsDescription(source->getOutputs().front().getHeader().getNamesAndTypesList()); + return ColumnsDescription(DistributedAsyncInsertSource(paths[0]).getOutputs().front().getHeader().getNamesAndTypesList()); } if (paths.empty() && !FormatFactory::instance().checkIfFormatHasExternalSchemaReader(format)) @@ -597,7 +596,7 @@ public: /// Special case for distributed format. Defaults are not needed here. if (storage->format_name == "Distributed") { - pipeline = std::make_unique(StorageDistributedDirectoryMonitor::createSourceFromFile(current_path)); + pipeline = std::make_unique(std::make_shared(current_path)); reader = std::make_unique(*pipeline); continue; } diff --git a/src/Storages/StorageJoin.cpp b/src/Storages/StorageJoin.cpp index ab1406f9bd6..dec741beb45 100644 --- a/src/Storages/StorageJoin.cpp +++ b/src/Storages/StorageJoin.cpp @@ -21,21 +21,24 @@ #include #include #include -#include /// toLower +#include +#include +namespace fs = std::filesystem; namespace DB { namespace ErrorCodes { - extern const int NOT_IMPLEMENTED; - extern const int LOGICAL_ERROR; - extern const int UNSUPPORTED_JOIN_KEYS; - extern const int NO_SUCH_COLUMN_IN_TABLE; - extern const int INCOMPATIBLE_TYPE_OF_JOIN; - extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; extern const int BAD_ARGUMENTS; + extern const int DEADLOCK_AVOIDED; + extern const int INCOMPATIBLE_TYPE_OF_JOIN; + extern const int LOGICAL_ERROR; + extern const int NO_SUCH_COLUMN_IN_TABLE; + extern const int NOT_IMPLEMENTED; + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; + extern const int UNSUPPORTED_JOIN_KEYS; } StorageJoin::StorageJoin( @@ -78,6 +81,14 @@ RWLockImpl::LockHolder StorageJoin::tryLockTimedWithContext(const RWLock & lock, return tryLockTimed(lock, type, query_id, acquire_timeout); } +RWLockImpl::LockHolder StorageJoin::tryLockForCurrentQueryTimedWithContext(const RWLock & lock, RWLockImpl::Type type, ContextPtr context) +{ + const String query_id = context ? context->getInitialQueryId() : RWLockImpl::NO_QUERY; + const std::chrono::milliseconds acquire_timeout + = context ? context->getSettingsRef().lock_acquire_timeout : std::chrono::seconds(DBMS_DEFAULT_LOCK_ACQUIRE_TIMEOUT_SEC); + return lock->getLock(type, query_id, acquire_timeout, false); +} + SinkToStoragePtr StorageJoin::write(const ASTPtr & query, const StorageMetadataPtr & metadata_snapshot, ContextPtr context) { std::lock_guard mutate_lock(mutate_mutex); @@ -95,7 +106,7 @@ void StorageJoin::truncate(const ASTPtr &, const StorageMetadataPtr &, ContextPt LOG_INFO(&Poco::Logger::get("StorageJoin"), "Path {} is already removed from disk {}", path, disk->getName()); disk->createDirectories(path); - disk->createDirectories(path + "tmp/"); + disk->createDirectories(fs::path(path) / "tmp/"); increment = 0; join = std::make_shared(table_join, getRightSampleBlock(), overwrite); @@ -238,8 +249,12 @@ void StorageJoin::insertBlock(const Block & block, ContextPtr context) { Block block_to_insert = block; convertRightBlock(block_to_insert); + TableLockHolder holder = tryLockForCurrentQueryTimedWithContext(rwlock, RWLockImpl::Write, context); + + /// Protection from `INSERT INTO test_table_join SELECT * FROM test_table_join` + if (!holder) + throw Exception(ErrorCodes::DEADLOCK_AVOIDED, "StorageJoin: cannot insert data because current query tries to read from this storage"); - TableLockHolder holder = tryLockTimedWithContext(rwlock, RWLockImpl::Write, context); join->addJoinedBlock(block_to_insert, true); } diff --git a/src/Storages/StorageJoin.h b/src/Storages/StorageJoin.h index 61ea743c841..a5e85d8788a 100644 --- a/src/Storages/StorageJoin.h +++ b/src/Storages/StorageJoin.h @@ -100,12 +100,15 @@ private: /// Protect state for concurrent use in insertFromBlock and joinBlock. /// Lock is stored in HashJoin instance during query and blocks concurrent insertions. mutable RWLock rwlock = RWLockImpl::create(); + mutable std::mutex mutate_mutex; void insertBlock(const Block & block, ContextPtr context) override; void finishInsert() override {} size_t getSize(ContextPtr context) const override; RWLockImpl::LockHolder tryLockTimedWithContext(const RWLock & lock, RWLockImpl::Type type, ContextPtr context) const; + /// Same as tryLockTimedWithContext, but returns `nullptr` if lock is already acquired by current query. + static RWLockImpl::LockHolder tryLockForCurrentQueryTimedWithContext(const RWLock & lock, RWLockImpl::Type type, ContextPtr context); void convertRightBlock(Block & block) const; }; diff --git a/src/Storages/StorageLog.cpp b/src/Storages/StorageLog.cpp index 338fb54c7e5..772ed34b7a9 100644 --- a/src/Storages/StorageLog.cpp +++ b/src/Storages/StorageLog.cpp @@ -118,7 +118,7 @@ private: if (limited_by_file_size) { - limited.emplace(*plain, file_size - offset, false); + limited.emplace(*plain, file_size - offset, /* trow_exception */ false, /* exact_limit */ std::optional()); compressed.emplace(*limited); } else @@ -1027,11 +1027,7 @@ void StorageLog::restoreDataImpl(const BackupPtr & backup, const String & data_p if (!backup->fileExists(file_path_in_backup)) throw Exception(ErrorCodes::CANNOT_RESTORE_TABLE, "File {} in backup is required to restore table", file_path_in_backup); - auto backup_entry = backup->readFile(file_path_in_backup); - auto in = backup_entry->getReadBuffer(); - auto out = disk->writeFile(data_file.path, max_compress_block_size, WriteMode::Append); - copyData(*in, *out); - out->finalize(); + backup->copyFileToDisk(file_path_in_backup, disk, data_file.path, WriteMode::Append); } if (use_marks_file) @@ -1062,8 +1058,7 @@ void StorageLog::restoreDataImpl(const BackupPtr & backup, const String & data_p old_num_rows[i] = num_marks ? data_files[i].marks[num_marks - 1].rows : 0; } - auto backup_entry = backup->readFile(file_path_in_backup); - auto marks_rb = backup_entry->getReadBuffer(); + auto marks_rb = backup->readFile(file_path_in_backup); for (size_t i = 0; i != num_extra_marks; ++i) { diff --git a/src/Storages/StorageMemory.cpp b/src/Storages/StorageMemory.cpp index 881cbc18b10..e7bd7cd8e7e 100644 --- a/src/Storages/StorageMemory.cpp +++ b/src/Storages/StorageMemory.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include @@ -192,6 +193,83 @@ private: }; +class ReadFromMemoryStorageStep final : public ISourceStep +{ +public: + explicit ReadFromMemoryStorageStep(Pipe pipe_) : + ISourceStep(DataStream{.header = pipe_.getHeader()}), + pipe(std::move(pipe_)) + { + } + + ReadFromMemoryStorageStep() = delete; + ReadFromMemoryStorageStep(const ReadFromMemoryStorageStep &) = delete; + ReadFromMemoryStorageStep & operator=(const ReadFromMemoryStorageStep &) = delete; + + ReadFromMemoryStorageStep(ReadFromMemoryStorageStep &&) = default; + ReadFromMemoryStorageStep & operator=(ReadFromMemoryStorageStep &&) = default; + + String getName() const override { return name; } + + void initializePipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &) override + { + // use move - make sure that the call will only be made once. + pipeline.init(std::move(pipe)); + } + + static Pipe makePipe(const Names & columns_to_read_, + const StorageSnapshotPtr & storage_snapshot_, + size_t num_streams_, + const bool delay_read_for_global_subqueries_) + { + storage_snapshot_->check(columns_to_read_); + + const auto & snapshot_data = assert_cast(*storage_snapshot_->data); + auto current_data = snapshot_data.blocks; + + if (delay_read_for_global_subqueries_) + { + /// Note: for global subquery we use single source. + /// Mainly, the reason is that at this point table is empty, + /// and we don't know the number of blocks are going to be inserted into it. + /// + /// It may seem to be not optimal, but actually data from such table is used to fill + /// set for IN or hash table for JOIN, which can't be done concurrently. + /// Since no other manipulation with data is done, multiple sources shouldn't give any profit. + + return Pipe(std::make_shared( + columns_to_read_, + storage_snapshot_, + nullptr /* data */, + nullptr /* parallel execution index */, + [current_data](std::shared_ptr & data_to_initialize) + { + data_to_initialize = current_data; + })); + } + + size_t size = current_data->size(); + + if (num_streams_ > size) + num_streams_ = size; + + Pipes pipes; + + auto parallel_execution_index = std::make_shared>(0); + + for (size_t stream = 0; stream < num_streams_; ++stream) + { + pipes.emplace_back(std::make_shared(columns_to_read_, storage_snapshot_, current_data, parallel_execution_index)); + } + return Pipe::unitePipes(std::move(pipes)); + } + +private: + static constexpr auto name = "ReadFromMemoryStorage"; + Pipe pipe; +}; + + StorageMemory::StorageMemory( const StorageID & table_id_, ColumnsDescription columns_description_, @@ -233,47 +311,29 @@ Pipe StorageMemory::read( size_t /*max_block_size*/, size_t num_streams) { - storage_snapshot->check(column_names); + return ReadFromMemoryStorageStep::makePipe(column_names, storage_snapshot, num_streams, delay_read_for_global_subqueries); +} - const auto & snapshot_data = assert_cast(*storage_snapshot->data); - auto current_data = snapshot_data.blocks; - - if (delay_read_for_global_subqueries) +void StorageMemory::read( + QueryPlan & query_plan, + const Names & column_names, + const StorageSnapshotPtr & storage_snapshot, + SelectQueryInfo & query_info, + ContextPtr context, + QueryProcessingStage::Enum processed_stage, + size_t max_block_size, + size_t num_streams) +{ + // @TODO it looks like IStorage::readFromPipe. different only step's type. + auto pipe = read(column_names, storage_snapshot, query_info, context, processed_stage, max_block_size, num_streams); + if (pipe.empty()) { - /// Note: for global subquery we use single source. - /// Mainly, the reason is that at this point table is empty, - /// and we don't know the number of blocks are going to be inserted into it. - /// - /// It may seem to be not optimal, but actually data from such table is used to fill - /// set for IN or hash table for JOIN, which can't be done concurrently. - /// Since no other manipulation with data is done, multiple sources shouldn't give any profit. - - return Pipe(std::make_shared( - column_names, - storage_snapshot, - nullptr /* data */, - nullptr /* parallel execution index */, - [current_data](std::shared_ptr & data_to_initialize) - { - data_to_initialize = current_data; - })); + auto header = storage_snapshot->getSampleBlockForColumns(column_names); + InterpreterSelectQuery::addEmptySourceToQueryPlan(query_plan, header, query_info, context); + return; } - - size_t size = current_data->size(); - - if (num_streams > size) - num_streams = size; - - Pipes pipes; - - auto parallel_execution_index = std::make_shared>(0); - - for (size_t stream = 0; stream < num_streams; ++stream) - { - pipes.emplace_back(std::make_shared(column_names, storage_snapshot, current_data, parallel_execution_index)); - } - - return Pipe::unitePipes(std::move(pipes)); + auto read_step = std::make_unique(std::move(pipe)); + query_plan.addStep(std::move(read_step)); } @@ -535,8 +595,7 @@ void StorageMemory::restoreDataImpl(const BackupPtr & backup, const String & dat if (!backup->fileExists(index_file_path)) throw Exception(ErrorCodes::CANNOT_RESTORE_TABLE, "File {} in backup is required to restore table", index_file_path); - auto backup_entry = backup->readFile(index_file_path); - auto in = backup_entry->getReadBuffer(); + auto in = backup->readFile(index_file_path); CompressedReadBuffer compressed_in{*in}; index.read(compressed_in); } @@ -550,8 +609,7 @@ void StorageMemory::restoreDataImpl(const BackupPtr & backup, const String & dat if (!backup->fileExists(data_file_path)) throw Exception(ErrorCodes::CANNOT_RESTORE_TABLE, "File {} in backup is required to restore table", data_file_path); - auto backup_entry = backup->readFile(data_file_path); - std::unique_ptr in = backup_entry->getReadBuffer(); + auto in = backup->readFile(data_file_path); std::optional temp_data_file; if (!dynamic_cast(in.get())) { diff --git a/src/Storages/StorageMemory.h b/src/Storages/StorageMemory.h index c739088dbe4..fcaadaff3ba 100644 --- a/src/Storages/StorageMemory.h +++ b/src/Storages/StorageMemory.h @@ -53,6 +53,16 @@ public: size_t max_block_size, size_t num_streams) override; + void read( + QueryPlan & query_plan, + const Names & column_names, + const StorageSnapshotPtr & storage_snapshot, + SelectQueryInfo & query_info, + ContextPtr context, + QueryProcessingStage::Enum processed_stage, + size_t max_block_size, + size_t num_streams) override; + bool supportsParallelInsert() const override { return true; } bool supportsSubcolumns() const override { return true; } bool supportsDynamicSubcolumns() const override { return true; } diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index af5a32d47f7..0f7b47255f6 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -448,7 +448,7 @@ void ReadFromMerge::initializePipeline(QueryPipelineBuilder & pipeline, const Bu size_t current_need_streams = tables_count >= num_streams ? 1 : (num_streams / tables_count); size_t current_streams = std::min(current_need_streams, remaining_streams); remaining_streams -= current_streams; - current_streams = std::max(static_cast(1), current_streams); + current_streams = std::max(1uz, current_streams); const auto & storage = std::get<1>(table); diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index aadd7b8c20a..d9bb189524c 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -223,8 +224,12 @@ void StorageMergeTree::read( auto cluster = local_context->getCluster(local_context->getSettingsRef().cluster_for_parallel_replicas); - Block header = - InterpreterSelectQuery(modified_query_ast, local_context, SelectQueryOptions(processed_stage).analyze()).getSampleBlock(); + Block header; + + if (local_context->getSettingsRef().allow_experimental_analyzer) + header = InterpreterSelectQueryAnalyzer::getSampleBlock(modified_query_ast, local_context, SelectQueryOptions(processed_stage).analyze()); + else + header = InterpreterSelectQuery(modified_query_ast, local_context, SelectQueryOptions(processed_stage).analyze()).getSampleBlock(); ClusterProxy::SelectStreamFactory select_stream_factory = ClusterProxy::SelectStreamFactory( diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 429339386b4..5f1c0810291 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -3,7 +3,7 @@ #include #include -#include +#include #include #include #include @@ -796,7 +796,7 @@ bool StorageReplicatedMergeTree::createTableIfNotExists(const StorageMetadataPtr auto code = zookeeper->tryMulti(ops, responses); if (code == Coordination::Error::ZNODEEXISTS) { - LOG_WARNING(log, "It looks like the table {} was created by another server at the same moment, will retry", zookeeper_path); + LOG_INFO(log, "It looks like the table {} was created by another server at the same moment, will retry", zookeeper_path); continue; } else if (code != Coordination::Error::ZOK) @@ -876,7 +876,7 @@ void StorageReplicatedMergeTree::createReplica(const StorageMetadataPtr & metada case Coordination::Error::ZNODEEXISTS: throw Exception(ErrorCodes::REPLICA_ALREADY_EXISTS, "Replica {} already exists", replica_path); case Coordination::Error::ZBADVERSION: - LOG_ERROR(log, "Retrying createReplica(), because some other replicas were created at the same time"); + LOG_INFO(log, "Retrying createReplica(), because some other replicas were created at the same time"); break; case Coordination::Error::ZNONODE: throw Exception(ErrorCodes::ALL_REPLICAS_LOST, "Table {} was suddenly removed", zookeeper_path); @@ -2951,7 +2951,8 @@ void StorageReplicatedMergeTree::cloneReplicaIfNeeded(zkutil::ZooKeeperPtr zooke } if (source_replica.empty()) - throw Exception(ErrorCodes::ALL_REPLICAS_LOST, "All replicas are lost"); + throw Exception(ErrorCodes::ALL_REPLICAS_LOST, "All replicas are lost. " + "See SYSTEM DROP REPLICA and SYSTEM RESTORE REPLICA queries, they may help"); if (is_new_replica) LOG_INFO(log, "Will mimic {}", source_replica); @@ -8434,7 +8435,11 @@ std::pair StorageReplicatedMergeTree::unlockSharedDataByID( } else if (error_code == Coordination::Error::ZNONODE) { - LOG_TRACE(logger, "Node with parent zookeeper lock {} for part {} doesn't exist (part was unlocked before)", zookeeper_part_uniq_node, part_name); + /// We don't know what to do, because this part can be mutation part + /// with hardlinked columns. Since we don't have this information (about blobs not to remove) + /// we refuse to remove blobs. + LOG_WARNING(logger, "Node with parent zookeeper lock {} for part {} doesn't exist (part was unlocked before), refuse to remove blobs", zookeeper_part_uniq_node, part_name); + return {false, {}}; } else { @@ -8594,11 +8599,37 @@ std::optional StorageReplicatedMergeTree::getZeroCopyPartPath(const Stri return getZeroCopyPartPath(*getSettings(), toString(disk->getDataSourceDescription().type), getTableSharedID(), part_name, zookeeper_path)[0]; } +bool StorageReplicatedMergeTree::waitZeroCopyLockToDisappear(const ZeroCopyLock & lock, size_t milliseconds_to_wait) +{ + if (lock.isLocked()) + return true; + + if (partial_shutdown_called.load(std::memory_order_relaxed)) + return true; + + auto lock_path = lock.lock->getLockPath(); + zkutil::ZooKeeperPtr zookeeper = tryGetZooKeeper(); + if (!zookeeper) + return true; + + Stopwatch time_waiting; + const auto & stop_waiting = [&]() + { + bool timeout_exceeded = milliseconds_to_wait < time_waiting.elapsedMilliseconds(); + return partial_shutdown_called.load(std::memory_order_relaxed) || is_readonly.load(std::memory_order_relaxed) || timeout_exceeded; + }; + + return zookeeper->waitForDisappear(lock_path, stop_waiting); +} + std::optional StorageReplicatedMergeTree::tryCreateZeroCopyExclusiveLock(const String & part_name, const DiskPtr & disk) { if (!disk || !disk->supportZeroCopyReplication()) return std::nullopt; + if (partial_shutdown_called.load(std::memory_order_relaxed) || is_readonly.load(std::memory_order_relaxed)) + return std::nullopt; + zkutil::ZooKeeperPtr zookeeper = tryGetZooKeeper(); if (!zookeeper) return std::nullopt; @@ -8611,10 +8642,8 @@ std::optional StorageReplicatedMergeTree::tryCreateZeroCopyExclusi /// Create actual lock ZeroCopyLock lock(zookeeper, zc_zookeeper_path, replica_name); - if (lock.lock->tryLock()) - return lock; - else - return std::nullopt; + lock.lock->tryLock(); + return lock; } String StorageReplicatedMergeTree::findReplicaHavingPart( diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h index 46c78e9064a..badc976b0cb 100644 --- a/src/Storages/StorageReplicatedMergeTree.h +++ b/src/Storages/StorageReplicatedMergeTree.h @@ -866,9 +866,14 @@ private: std::optional getZeroCopyPartPath(const String & part_name, const DiskPtr & disk); /// Create ephemeral lock in zookeeper for part and disk which support zero copy replication. - /// If somebody already holding the lock -- return std::nullopt. + /// If no connection to zookeeper, shutdown, readonly -- return std::nullopt. + /// If somebody already holding the lock -- return unlocked ZeroCopyLock object (not std::nullopt). std::optional tryCreateZeroCopyExclusiveLock(const String & part_name, const DiskPtr & disk) override; + /// Wait for ephemral lock to disappear. Return true if table shutdown/readonly/timeout exceeded, etc. + /// Or if node actually disappeared. + bool waitZeroCopyLockToDisappear(const ZeroCopyLock & lock, size_t milliseconds_to_wait) override; + void startupImpl(bool from_attach_thread); }; diff --git a/src/Storages/StorageS3Cluster.cpp b/src/Storages/StorageS3Cluster.cpp index c7535bb4550..51a4a311e93 100644 --- a/src/Storages/StorageS3Cluster.cpp +++ b/src/Storages/StorageS3Cluster.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -102,7 +103,20 @@ Pipe StorageS3Cluster::read( auto extension = getTaskIteratorExtension(query_info.query, context); /// Calculate the header. This is significant, because some columns could be thrown away in some cases like query with count(*) - auto interpreter = InterpreterSelectQuery(query_info.query, context, SelectQueryOptions(processed_stage).analyze()); + + Block sample_block; + ASTPtr query_to_send = query_info.query; + + if (context->getSettingsRef().allow_experimental_analyzer) + { + sample_block = InterpreterSelectQueryAnalyzer::getSampleBlock(query_info.query, context, SelectQueryOptions(processed_stage)); + } + else + { + auto interpreter = InterpreterSelectQuery(query_info.query, context, SelectQueryOptions(processed_stage).analyze()); + sample_block = interpreter.getSampleBlock(); + query_to_send = interpreter.getQueryInfo().query->clone(); + } const Scalars & scalars = context->hasQueryContext() ? context->getQueryContext()->getScalars() : Scalars{}; @@ -110,7 +124,6 @@ Pipe StorageS3Cluster::read( const bool add_agg_info = processed_stage == QueryProcessingStage::WithMergeableState; - ASTPtr query_to_send = interpreter.getQueryInfo().query->clone(); if (!structure_argument_was_provided) addColumnsStructureToQueryWithClusterEngine( query_to_send, StorageDictionary::generateNamesAndTypesDescription(storage_snapshot->metadata->getColumns().getAll()), 5, getName()); @@ -133,10 +146,9 @@ Pipe StorageS3Cluster::read( for (auto & try_result : try_results) { auto remote_query_executor = std::make_shared( - shard_info.pool, std::vector{try_result}, queryToString(query_to_send), - interpreter.getSampleBlock(), + sample_block, context, /*throttler=*/nullptr, scalars, diff --git a/src/Storages/StorageStripeLog.cpp b/src/Storages/StorageStripeLog.cpp index 870f6b96ae6..30585250be2 100644 --- a/src/Storages/StorageStripeLog.cpp +++ b/src/Storages/StorageStripeLog.cpp @@ -622,11 +622,7 @@ void StorageStripeLog::restoreDataImpl(const BackupPtr & backup, const String & if (!backup->fileExists(file_path_in_backup)) throw Exception(ErrorCodes::CANNOT_RESTORE_TABLE, "File {} in backup is required to restore table", file_path_in_backup); - auto backup_entry = backup->readFile(file_path_in_backup); - auto in = backup_entry->getReadBuffer(); - auto out = disk->writeFile(data_file_path, max_compress_block_size, WriteMode::Append); - copyData(*in, *out); - out->finalize(); + backup->copyFileToDisk(file_path_in_backup, disk, data_file_path, WriteMode::Append); } /// Append the index. @@ -636,8 +632,7 @@ void StorageStripeLog::restoreDataImpl(const BackupPtr & backup, const String & if (!backup->fileExists(index_path_in_backup)) throw Exception(ErrorCodes::CANNOT_RESTORE_TABLE, "File {} in backup is required to restore table", index_path_in_backup); - auto backup_entry = backup->readFile(index_path_in_backup); - auto index_in = backup_entry->getReadBuffer(); + auto index_in = backup->readFile(index_path_in_backup); CompressedReadBuffer index_compressed_in{*index_in}; extra_indices.read(index_compressed_in); diff --git a/src/Storages/StorageURL.cpp b/src/Storages/StorageURL.cpp index 691254867ac..982e7fb5f9b 100644 --- a/src/Storages/StorageURL.cpp +++ b/src/Storages/StorageURL.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include @@ -17,6 +18,7 @@ #include #include #include +#include #include #include @@ -30,6 +32,7 @@ #include #include #include +#include #include #include @@ -217,6 +220,15 @@ namespace uri_options.size() == 1, download_threads); + try + { + total_size += getFileSizeFromReadBuffer(*read_buf); + } + catch (...) + { + // we simply continue without total_size + } + auto input_format = FormatFactory::instance().getInput(format, *read_buf, sample_block, context, max_block_size, format_settings); QueryPipelineBuilder builder; @@ -257,7 +269,14 @@ namespace Chunk chunk; if (reader->pull(chunk)) + { + UInt64 num_rows = chunk.getNumRows(); + if (num_rows && total_size) + updateRowsProgressApprox( + *this, chunk, total_size, total_rows_approx_accumulated, total_rows_count_times, total_rows_approx_max); + return chunk; + } pipeline->reset(); reader.reset(); @@ -447,6 +466,11 @@ namespace std::unique_ptr reader; Poco::Net::HTTPBasicCredentials credentials; + + size_t total_size = 0; + UInt64 total_rows_approx_max = 0; + size_t total_rows_count_times = 0; + UInt64 total_rows_approx_accumulated = 0; }; } @@ -458,6 +482,7 @@ StorageURLSink::StorageURLSink( ContextPtr context, const ConnectionTimeouts & timeouts, const CompressionMethod compression_method, + const HTTPHeaderEntries & headers, const String & http_method) : SinkToStorage(sample_block) { @@ -465,7 +490,7 @@ StorageURLSink::StorageURLSink( std::string content_encoding = toContentEncodingName(compression_method); write_buf = wrapWriteBufferWithCompressionMethod( - std::make_unique(Poco::URI(uri), http_method, content_type, content_encoding, timeouts), + std::make_unique(Poco::URI(uri), http_method, content_type, content_encoding, headers, timeouts), compression_method, 3); writer = FormatFactory::instance().getOutputFormat(format, *write_buf, sample_block, context, format_settings); @@ -530,6 +555,7 @@ public: ContextPtr context_, const ConnectionTimeouts & timeouts_, const CompressionMethod compression_method_, + const HTTPHeaderEntries & headers_, const String & http_method_) : PartitionedSink(partition_by, context_, sample_block_) , uri(uri_) @@ -539,6 +565,7 @@ public: , context(context_) , timeouts(timeouts_) , compression_method(compression_method_) + , headers(headers_) , http_method(http_method_) { } @@ -548,7 +575,7 @@ public: auto partition_path = PartitionedSink::replaceWildcards(uri, partition_id); context->getRemoteHostFilter().checkURL(Poco::URI(partition_path)); return std::make_shared( - partition_path, format, format_settings, sample_block, context, timeouts, compression_method, http_method); + partition_path, format, format_settings, sample_block, context, timeouts, compression_method, headers, http_method); } private: @@ -560,6 +587,7 @@ private: const ConnectionTimeouts timeouts; const CompressionMethod compression_method; + const HTTPHeaderEntries headers; const String http_method; }; @@ -821,6 +849,7 @@ SinkToStoragePtr IStorageURLBase::write(const ASTPtr & query, const StorageMetad context, getHTTPTimeouts(context), compression_method, + headers, http_method); } else @@ -833,6 +862,7 @@ SinkToStoragePtr IStorageURLBase::write(const ASTPtr & query, const StorageMetad context, getHTTPTimeouts(context), compression_method, + headers, http_method); } } diff --git a/src/Storages/StorageURL.h b/src/Storages/StorageURL.h index c95cfa69e54..65ee78e1e73 100644 --- a/src/Storages/StorageURL.h +++ b/src/Storages/StorageURL.h @@ -136,6 +136,7 @@ public: ContextPtr context, const ConnectionTimeouts & timeouts, CompressionMethod compression_method, + const HTTPHeaderEntries & headers = {}, const String & method = Poco::Net::HTTPRequest::HTTP_POST); std::string getName() const override { return "StorageURLSink"; } diff --git a/src/Storages/StorageView.cpp b/src/Storages/StorageView.cpp index 1a7050b4dff..470def7e197 100644 --- a/src/Storages/StorageView.cpp +++ b/src/Storages/StorageView.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include @@ -36,6 +37,7 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } + namespace { @@ -117,8 +119,12 @@ StorageView::StorageView( SelectQueryDescription description; description.inner_query = query.select->ptr(); + + NormalizeSelectWithUnionQueryVisitor::Data data{SetOperationMode::Unspecified}; + NormalizeSelectWithUnionQueryVisitor{data}.visit(description.inner_query); + is_parameterized_view = query.isParameterizedView(); - parameter_types = analyzeReceiveQueryParamsWithType(description.inner_query); + view_parameter_types = analyzeReceiveQueryParamsWithType(description.inner_query); storage_metadata.setSelectQuery(description); setInMemoryMetadata(storage_metadata); } @@ -167,7 +173,7 @@ void StorageView::read( query_plan.addStep(std::move(materializing)); /// And also convert to expected structure. - const auto & expected_header = storage_snapshot->getSampleBlockForColumns(column_names,parameter_values); + const auto & expected_header = storage_snapshot->getSampleBlockForColumns(column_names, query_info.parameterized_view_values); const auto & header = query_plan.getCurrentDataStream().header; const auto * select_with_union = current_inner_query->as(); @@ -203,7 +209,7 @@ static ASTTableExpression * getFirstTableExpression(ASTSelectQuery & select_quer return select_element->table_expression->as(); } -void StorageView::replaceQueryParametersIfParametrizedView(ASTPtr & outer_query) +void StorageView::replaceQueryParametersIfParametrizedView(ASTPtr & outer_query, const NameToNameMap & parameter_values) { ReplaceQueryParameterVisitor visitor(parameter_values); visitor.visit(outer_query); @@ -261,7 +267,8 @@ String StorageView::replaceQueryParameterWithValue(const String & column_name, c if ((pos = name.find(parameter.first)) != std::string::npos) { auto parameter_datatype_iterator = parameter_types.find(parameter.first); - if (parameter_datatype_iterator != parameter_types.end()) + size_t parameter_end = pos + parameter.first.size(); + if (parameter_datatype_iterator != parameter_types.end() && name.size() >= parameter_end && (name[parameter_end] == ',' || name[parameter_end] == ')')) { String parameter_name("_CAST(" + parameter.second + ", '" + parameter_datatype_iterator->second + "')"); name.replace(pos, parameter.first.size(), parameter_name); diff --git a/src/Storages/StorageView.h b/src/Storages/StorageView.h index 6cd4bb171f5..bebecb79ec0 100644 --- a/src/Storages/StorageView.h +++ b/src/Storages/StorageView.h @@ -35,7 +35,7 @@ public: size_t max_block_size, size_t num_streams) override; - void replaceQueryParametersIfParametrizedView(ASTPtr & outer_query); + static void replaceQueryParametersIfParametrizedView(ASTPtr & outer_query, const NameToNameMap & parameter_values); static void replaceWithSubquery(ASTSelectQuery & select_query, ASTPtr & view_name, const StorageMetadataPtr & metadata_snapshot, const bool parameterized_view) { @@ -47,20 +47,14 @@ public: static String replaceQueryParameterWithValue (const String & column_name, const NameToNameMap & parameter_values, const NameToNameMap & parameter_types); static String replaceValueWithQueryParameter (const String & column_name, const NameToNameMap & parameter_values); - void setParameterValues (NameToNameMap parameter_values_) + const NameToNameMap & getParameterTypes() const { - parameter_values = parameter_values_; - } - - NameToNameMap getParameterValues() const - { - return parameter_types; + return view_parameter_types; } protected: bool is_parameterized_view; - NameToNameMap parameter_values; - NameToNameMap parameter_types; + NameToNameMap view_parameter_types; }; } diff --git a/src/Storages/System/StorageSystemDistributionQueue.cpp b/src/Storages/System/StorageSystemDistributionQueue.cpp index 34cff7df65d..d57269f0638 100644 --- a/src/Storages/System/StorageSystemDistributionQueue.cpp +++ b/src/Storages/System/StorageSystemDistributionQueue.cpp @@ -3,7 +3,6 @@ #include #include #include -#include #include #include #include @@ -174,7 +173,7 @@ void StorageSystemDistributionQueue::fillData(MutableColumns & res_columns, Cont auto & distributed_table = dynamic_cast(*tables[database][table]); - for (const auto & status : distributed_table.getDirectoryMonitorsStatuses()) + for (const auto & status : distributed_table.getDirectoryQueueStatuses()) { size_t col_num = 0; res_columns[col_num++]->insert(database); diff --git a/src/Storages/System/StorageSystemMarkedDroppedTables.cpp b/src/Storages/System/StorageSystemMarkedDroppedTables.cpp new file mode 100644 index 00000000000..fcdd6e1edcf --- /dev/null +++ b/src/Storages/System/StorageSystemMarkedDroppedTables.cpp @@ -0,0 +1,64 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "base/types.h" + + +namespace DB +{ + +NamesAndTypesList StorageSystemMarkedDroppedTables::getNamesAndTypes() +{ + NamesAndTypesList names_and_types{ + {"index", std::make_shared()}, + {"database", std::make_shared()}, + {"table", std::make_shared()}, + {"uuid", std::make_shared()}, + {"engine", std::make_shared()}, + {"metadata_dropped_path", std::make_shared()}, + {"table_dropped_time", std::make_shared()}, + }; + return names_and_types; +} + + +void StorageSystemMarkedDroppedTables::fillData(MutableColumns & res_columns, ContextPtr, const SelectQueryInfo &) const +{ + auto tables_mark_dropped = DatabaseCatalog::instance().getTablesMarkedDropped(); + + size_t index = 0; + + auto & column_index = assert_cast(*res_columns[index++]); + auto & column_database = assert_cast(*res_columns[index++]); + auto & column_table = assert_cast(*res_columns[index++]); + auto & column_uuid = assert_cast(*res_columns[index++]).getData(); + auto & column_engine = assert_cast(*res_columns[index++]); + auto & column_metadata_dropped_path = assert_cast(*res_columns[index++]); + auto & column_table_dropped_time = assert_cast(*res_columns[index++]); + + auto add_row = [&](UInt32 idx, const DatabaseCatalog::TableMarkedAsDropped & table_mark_dropped) + { + column_index.insertValue(idx); + column_database.insertData(table_mark_dropped.table_id.getDatabaseName().data(), table_mark_dropped.table_id.getDatabaseName().size()); + column_table.insertData(table_mark_dropped.table_id.getTableName().data(), table_mark_dropped.table_id.getTableName().size()); + column_uuid.push_back(table_mark_dropped.table_id.uuid.toUnderType()); + if (table_mark_dropped.table) + column_engine.insertData(table_mark_dropped.table->getName().data(), table_mark_dropped.table->getName().size()); + else + column_engine.insertData({}, 0); + column_metadata_dropped_path.insertData(table_mark_dropped.metadata_path.data(), table_mark_dropped.metadata_path.size()); + column_table_dropped_time.insertValue(static_cast(table_mark_dropped.drop_time)); + }; + + UInt32 idx = 0; + for (const auto & table_mark_dropped : tables_mark_dropped) + add_row(idx++, table_mark_dropped); +} + +} diff --git a/src/Storages/System/StorageSystemMarkedDroppedTables.h b/src/Storages/System/StorageSystemMarkedDroppedTables.h new file mode 100644 index 00000000000..ea2a864311c --- /dev/null +++ b/src/Storages/System/StorageSystemMarkedDroppedTables.h @@ -0,0 +1,20 @@ +#pragma once + +#include + + +namespace DB +{ + +class StorageSystemMarkedDroppedTables final : public IStorageSystemOneBlock +{ +public: + std::string getName() const override { return "SystemMarkedDroppedTables"; } + static NamesAndTypesList getNamesAndTypes(); + +protected: + using IStorageSystemOneBlock::IStorageSystemOneBlock; + void fillData(MutableColumns & res_columns, ContextPtr context, const SelectQueryInfo &) const override; +}; + +} diff --git a/src/Storages/System/StorageSystemParts.cpp b/src/Storages/System/StorageSystemParts.cpp index f6854e7d5d0..86ecb336b51 100644 --- a/src/Storages/System/StorageSystemParts.cpp +++ b/src/Storages/System/StorageSystemParts.cpp @@ -14,7 +14,7 @@ #include #include #include -#include +#include #include #include diff --git a/src/Storages/System/StorageSystemProjectionParts.cpp b/src/Storages/System/StorageSystemProjectionParts.cpp index 37c62ba5eb0..d2c6c3ef287 100644 --- a/src/Storages/System/StorageSystemProjectionParts.cpp +++ b/src/Storages/System/StorageSystemProjectionParts.cpp @@ -11,7 +11,7 @@ #include #include #include -#include +#include namespace DB { diff --git a/src/Storages/System/attachSystemTables.cpp b/src/Storages/System/attachSystemTables.cpp index 61329ab834b..fd1cf2f1623 100644 --- a/src/Storages/System/attachSystemTables.cpp +++ b/src/Storages/System/attachSystemTables.cpp @@ -79,6 +79,7 @@ #include #include #include +#include #ifdef OS_LINUX #include @@ -140,6 +141,7 @@ void attachSystemTablesLocal(ContextPtr context, IDatabase & system_database) attach(context, system_database, "time_zones"); attach(context, system_database, "backups"); attach(context, system_database, "schema_inference_cache"); + attach(context, system_database, "marked_dropped_tables"); #ifdef OS_LINUX attach(context, system_database, "stack_trace"); #endif diff --git a/src/Storages/WindowView/StorageWindowView.cpp b/src/Storages/WindowView/StorageWindowView.cpp index 3a74fd5fc75..3471e4ea6bf 100644 --- a/src/Storages/WindowView/StorageWindowView.cpp +++ b/src/Storages/WindowView/StorageWindowView.cpp @@ -78,6 +78,7 @@ namespace ErrorCodes extern const int SUPPORT_IS_DISABLED; extern const int TABLE_WAS_NOT_DROPPED; extern const int NOT_IMPLEMENTED; + extern const int UNSUPPORTED_METHOD; } namespace @@ -1158,6 +1159,10 @@ StorageWindowView::StorageWindowView( , fire_signal_timeout_s(context_->getSettingsRef().wait_for_window_view_fire_signal_timeout.totalSeconds()) , clean_interval_usec(context_->getSettingsRef().window_view_clean_interval.totalMicroseconds()) { + if (context_->getSettingsRef().allow_experimental_analyzer) + throw Exception(ErrorCodes::UNSUPPORTED_METHOD, + "Experimental WINDOW VIEW feature is not supported with new infrastructure for query analysis (the setting 'allow_experimental_analyzer')"); + if (!query.select) throw Exception(ErrorCodes::INCORRECT_QUERY, "SELECT query is not specified for {}", getName()); diff --git a/src/Storages/removeGroupingFunctionSpecializations.cpp b/src/Storages/removeGroupingFunctionSpecializations.cpp new file mode 100644 index 00000000000..668596756a1 --- /dev/null +++ b/src/Storages/removeGroupingFunctionSpecializations.cpp @@ -0,0 +1,65 @@ +#include + +#include +#include +#include +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + +class GeneralizeGroupingFunctionForDistributedVisitor : public InDepthQueryTreeVisitor +{ +public: + static void visitImpl(QueryTreeNodePtr & node) + { + auto * function = node->as(); + if (!function) + return; + + const auto & function_name = function->getFunctionName(); + bool ordinary_grouping = function_name == "groupingOrdinary"; + + if (!ordinary_grouping + && function_name != "groupingForRollup" + && function_name != "groupingForCube" + && function_name != "groupingForGroupingSets") + return; + + + if (!ordinary_grouping) + { + auto & arguments = function->getArguments().getNodes(); + + if (arguments.empty()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Grouping function specialization must have arguments"); + auto * grouping_set_arg = arguments[0]->as(); + if (!grouping_set_arg || grouping_set_arg->getColumnName() != "__grouping_set") + throw Exception(ErrorCodes::LOGICAL_ERROR, + "The first argument of Grouping function specialization must be '__grouping_set' column but {} found", + arguments[0]->dumpTree()); + arguments.erase(arguments.begin()); + } + + // This node will be only converted to AST, so we don't need + // to pass the correct force_compatibility flag to FunctionGrouping. + auto function_adaptor = std::make_shared( + std::make_shared(false) + ); + function->resolveAsFunction(function_adaptor); + } +}; + +void removeGroupingFunctionSpecializations(QueryTreeNodePtr & node) +{ + GeneralizeGroupingFunctionForDistributedVisitor visitor; + visitor.visit(node); +} + +} diff --git a/src/Storages/removeGroupingFunctionSpecializations.h b/src/Storages/removeGroupingFunctionSpecializations.h new file mode 100644 index 00000000000..878b87abce7 --- /dev/null +++ b/src/Storages/removeGroupingFunctionSpecializations.h @@ -0,0 +1,10 @@ +#pragma once + +#include + +namespace DB +{ + +void removeGroupingFunctionSpecializations(QueryTreeNodePtr & node); + +} diff --git a/src/TableFunctions/ITableFunctionFileLike.cpp b/src/TableFunctions/ITableFunctionFileLike.cpp index 8cbffc10e5a..bbaf2b68418 100644 --- a/src/TableFunctions/ITableFunctionFileLike.cpp +++ b/src/TableFunctions/ITableFunctionFileLike.cpp @@ -7,7 +7,6 @@ #include #include -#include #include #include diff --git a/src/TableFunctions/TableFunctionExplain.cpp b/src/TableFunctions/TableFunctionExplain.cpp index 081aa4fc1dc..f93ae00cd7d 100644 --- a/src/TableFunctions/TableFunctionExplain.cpp +++ b/src/TableFunctions/TableFunctionExplain.cpp @@ -11,6 +11,8 @@ #include #include #include +#include +#include namespace DB { @@ -123,7 +125,10 @@ static Block executeMonoBlock(QueryPipeline & pipeline) StoragePtr TableFunctionExplain::executeImpl( const ASTPtr & /*ast_function*/, ContextPtr context, const std::string & table_name, ColumnsDescription /*cached_columns*/) const { - BlockIO blockio = getInterpreter(context).execute(); + /// To support settings inside explain subquery. + auto mutable_context = Context::createCopy(context); + InterpreterSetQuery::applySettingsFromQuery(query, mutable_context); + BlockIO blockio = getInterpreter(mutable_context).execute(); Block block = executeMonoBlock(blockio.pipeline); StorageID storage_id(getDatabaseName(), table_name); diff --git a/src/TableFunctions/TableFunctionMerge.cpp b/src/TableFunctions/TableFunctionMerge.cpp index 066caa8170d..586cee54085 100644 --- a/src/TableFunctions/TableFunctionMerge.cpp +++ b/src/TableFunctions/TableFunctionMerge.cpp @@ -53,7 +53,7 @@ std::vector TableFunctionMerge::skipAnalysisForArguments(const QueryTree result.push_back(i); } - return {0}; + return result; } void TableFunctionMerge::parseArguments(const ASTPtr & ast_function, ContextPtr context) diff --git a/src/TableFunctions/TableFunctionURL.cpp b/src/TableFunctions/TableFunctionURL.cpp index 468f949203d..d8c3e72efe1 100644 --- a/src/TableFunctions/TableFunctionURL.cpp +++ b/src/TableFunctions/TableFunctionURL.cpp @@ -9,6 +9,8 @@ #include #include #include +#include +#include #include #include #include @@ -26,6 +28,24 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; } +std::vector TableFunctionURL::skipAnalysisForArguments(const QueryTreeNodePtr & query_node_table_function, ContextPtr) const +{ + auto & table_function_node = query_node_table_function->as(); + auto & table_function_arguments_nodes = table_function_node.getArguments().getNodes(); + size_t table_function_arguments_size = table_function_arguments_nodes.size(); + + std::vector result; + + for (size_t i = 0; i < table_function_arguments_size; ++i) + { + auto * function_node = table_function_arguments_nodes[i]->as(); + if (function_node && function_node->getFunctionName() == "headers") + result.push_back(i); + } + + return result; +} + void TableFunctionURL::parseArguments(const ASTPtr & ast, ContextPtr context) { const auto & ast_function = assert_cast(ast.get()); diff --git a/src/TableFunctions/TableFunctionURL.h b/src/TableFunctions/TableFunctionURL.h index a670bdc0682..dca5123fb69 100644 --- a/src/TableFunctions/TableFunctionURL.h +++ b/src/TableFunctions/TableFunctionURL.h @@ -12,7 +12,7 @@ class Context; /* url(source, format[, structure, compression]) - creates a temporary storage from url. */ -class TableFunctionURL : public ITableFunctionFileLike +class TableFunctionURL final: public ITableFunctionFileLike { public: static constexpr auto name = "url"; @@ -23,10 +23,11 @@ public: ColumnsDescription getActualTableStructure(ContextPtr context) const override; -protected: +private: + std::vector skipAnalysisForArguments(const QueryTreeNodePtr & query_node_table_function, ContextPtr context) const override; + void parseArguments(const ASTPtr & ast, ContextPtr context) override; -private: StoragePtr getStorage( const String & source, const String & format_, const ColumnsDescription & columns, ContextPtr global_context, const std::string & table_name, const String & compression_method_) const override; diff --git a/tests/ci/autoscale_runners_lambda/app.py b/tests/ci/autoscale_runners_lambda/app.py new file mode 100644 index 00000000000..7e3af3f6779 --- /dev/null +++ b/tests/ci/autoscale_runners_lambda/app.py @@ -0,0 +1,245 @@ +#!/usr/bin/env python3 + +"""The lambda to decrease/increase ASG desired capacity based on current queue""" + +import json +import logging +import time +from dataclasses import dataclass +from pprint import pformat +from typing import Any, List, Literal, Optional, Tuple + +import boto3 # type: ignore +import requests # type: ignore + +RUNNER_TYPE_LABELS = [ + "builder", + "func-tester", + "func-tester-aarch64", + "fuzzer-unit-tester", + "stress-tester", + "style-checker", + "style-checker-aarch64", +] + +# 4 HOUR - is a balance to get the most precise values +# - Our longest possible running check is around 5h on the worst scenario +# - The long queue won't be wiped out and replaced, so the measurmenet is fine +# - If the data is spoiled by something, we are from the bills perspective +QUEUE_QUERY = f"""SELECT + last_status AS status, + toUInt32(count()) AS length, + labels +FROM +( + SELECT + arraySort(groupArray(status))[-1] AS last_status, + labels, + id, + html_url + FROM default.workflow_jobs + WHERE has(labels, 'self-hosted') + AND hasAny({RUNNER_TYPE_LABELS}, labels) + AND started_at > now() - INTERVAL 4 HOUR + GROUP BY ALL + HAVING last_status IN ('in_progress', 'queued') +) +GROUP BY ALL +ORDER BY labels, last_status""" + + +@dataclass +class Queue: + status: Literal["in_progress", "queued"] + lentgh: int + label: str + + +def get_scales(runner_type: str) -> Tuple[int, int]: + "returns the multipliers for scaling down and up ASG by types" + # Scaling down is quicker on the lack of running jobs than scaling up on + # queue + scale_down = 3 + scale_up = 5 + if runner_type == "style-checker": + # the style checkers have so many noise, so it scales up too quickly + scale_down = 2 + scale_up = 10 + return scale_down, scale_up + + +### VENDORING +def get_parameter_from_ssm(name, decrypt=True, client=None): + if not client: + client = boto3.client("ssm", region_name="us-east-1") + return client.get_parameter(Name=name, WithDecryption=decrypt)["Parameter"]["Value"] + + +class CHException(Exception): + pass + + +class ClickHouseHelper: + def __init__( + self, + url: Optional[str] = None, + user: Optional[str] = None, + password: Optional[str] = None, + ): + self.url = url + self.auth = {} + if user: + self.auth["X-ClickHouse-User"] = user + if password: + self.auth["X-ClickHouse-Key"] = password + + def _select_and_get_json_each_row(self, db, query): + params = { + "database": db, + "query": query, + "default_format": "JSONEachRow", + } + for i in range(5): + response = None + try: + response = requests.get(self.url, params=params, headers=self.auth) + response.raise_for_status() + return response.text + except Exception as ex: + logging.warning("Cannot fetch data with exception %s", str(ex)) + if response: + logging.warning("Reponse text %s", response.text) + time.sleep(0.1 * i) + + raise CHException("Cannot fetch data from clickhouse") + + def select_json_each_row(self, db, query): + text = self._select_and_get_json_each_row(db, query) + result = [] + for line in text.split("\n"): + if line: + result.append(json.loads(line)) + return result + + +CH_CLIENT = ClickHouseHelper(get_parameter_from_ssm("clickhouse-test-stat-url"), "play") + + +def set_capacity( + runner_type: str, queues: List[Queue], client: Any, dry_run: bool = True +) -> None: + assert len(queues) in (1, 2) + assert all(q.label == runner_type for q in queues) + as_groups = client.describe_auto_scaling_groups( + Filters=[ + {"Name": "tag-key", "Values": ["github:runner-type"]}, + {"Name": "tag-value", "Values": [runner_type]}, + ] + )["AutoScalingGroups"] + assert len(as_groups) == 1 + asg = as_groups[0] + running = 0 + queued = 0 + for q in queues: + if q.status == "in_progress": + running = q.lentgh + continue + if q.status == "queued": + queued = q.lentgh + continue + raise ValueError("Queue status is not in ['in_progress', 'queued']") + + scale_down, scale_up = get_scales(runner_type) + # How much nodes are free (positive) or need to be added (negative) + capacity_reserve = asg["DesiredCapacity"] - running - queued + stop = False + if capacity_reserve < 0: + # This part is about scaling up + capacity_deficit = -capacity_reserve + # It looks that we are still OK, since no queued jobs exist + stop = stop or queued == 0 + # Are we already at the capacity limits + stop = stop or asg["MaxSize"] <= asg["DesiredCapacity"] + # Let's calculate a new desired capacity + desired_capacity = asg["DesiredCapacity"] + (capacity_deficit // scale_up) + desired_capacity = max(desired_capacity, asg["MinSize"]) + desired_capacity = min(desired_capacity, asg["MaxSize"]) + # Finally, should the capacity be even changed + stop = stop or asg["DesiredCapacity"] == desired_capacity + if stop: + return + logging.info( + "The ASG %s capacity will be increased to %s, current capacity=%s, " + "maximum capacity=%s, running jobs=%s, queue size=%s", + asg["AutoScalingGroupName"], + desired_capacity, + asg["DesiredCapacity"], + asg["MaxSize"], + running, + queued, + ) + if not dry_run: + client.set_desired_capacity( + AutoScalingGroupName=asg["AutoScalingGroupName"], + DesiredCapacity=desired_capacity, + ) + return + + # Now we will calculate if we need to scale down + stop = stop or asg["DesiredCapacity"] == asg["MinSize"] + desired_capacity = asg["DesiredCapacity"] - (capacity_reserve // scale_down) + desired_capacity = max(desired_capacity, asg["MinSize"]) + desired_capacity = min(desired_capacity, asg["MaxSize"]) + stop = stop or asg["DesiredCapacity"] == desired_capacity + if stop: + return + + logging.info( + "The ASG %s capacity will be decreased to %s, current capacity=%s, " + "minimum capacity=%s, running jobs=%s, queue size=%s", + asg["AutoScalingGroupName"], + desired_capacity, + asg["DesiredCapacity"], + asg["MinSize"], + running, + queued, + ) + if not dry_run: + client.set_desired_capacity( + AutoScalingGroupName=asg["AutoScalingGroupName"], + DesiredCapacity=desired_capacity, + ) + + +def main(dry_run: bool = True) -> None: + logging.getLogger().setLevel(logging.INFO) + asg_client = boto3.client("autoscaling") + try: + global CH_CLIENT + queues = CH_CLIENT.select_json_each_row("default", QUEUE_QUERY) + except CHException as ex: + logging.exception( + "Got an exception on insert, tryuing to update the client " + "credentials and repeat", + exc_info=ex, + ) + CH_CLIENT = ClickHouseHelper( + get_parameter_from_ssm("clickhouse-test-stat-url"), "play" + ) + queues = CH_CLIENT.select_json_each_row("default", QUEUE_QUERY) + + logging.info("Received queue data:\n%s", pformat(queues, width=120)) + for runner_type in RUNNER_TYPE_LABELS: + runner_queues = [ + Queue(queue["status"], queue["length"], runner_type) + for queue in queues + if runner_type in queue["labels"] + ] + runner_queues = runner_queues or [Queue("in_progress", 0, runner_type)] + set_capacity(runner_type, runner_queues, asg_client, dry_run) + + +def handler(event: dict, context: Any) -> None: + _ = event + _ = context + return main(False) diff --git a/tests/ci/autoscale_runners_lambda/build_and_deploy_archive.sh b/tests/ci/autoscale_runners_lambda/build_and_deploy_archive.sh new file mode 120000 index 00000000000..96ba3fa024e --- /dev/null +++ b/tests/ci/autoscale_runners_lambda/build_and_deploy_archive.sh @@ -0,0 +1 @@ +../team_keys_lambda/build_and_deploy_archive.sh \ No newline at end of file diff --git a/tests/ci/autoscale_runners_lambda/requirements.txt b/tests/ci/autoscale_runners_lambda/requirements.txt new file mode 100644 index 00000000000..f2293605cf1 --- /dev/null +++ b/tests/ci/autoscale_runners_lambda/requirements.txt @@ -0,0 +1 @@ +requests diff --git a/tests/ci/autoscale_runners_lambda_test.py b/tests/ci/autoscale_runners_lambda_test.py new file mode 100644 index 00000000000..7efa0004745 --- /dev/null +++ b/tests/ci/autoscale_runners_lambda_test.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python + +import unittest +from dataclasses import dataclass +from typing import Any, List + +from autoscale_runners_lambda.app import set_capacity, Queue + + +@dataclass +class TestCase: + name: str + min_size: int + desired_capacity: int + max_size: int + queues: List[Queue] + expected_capacity: int + + +class TestSetCapacity(unittest.TestCase): + class FakeClient: + def __init__(self): + self._expected_data = {} # type: dict + self._expected_capacity = -1 + + @property + def expected_data(self) -> dict: + """a one-time property""" + data, self._expected_data = self._expected_data, {} + return data + + @expected_data.setter + def expected_data(self, value: dict) -> None: + self._expected_data = value + + @property + def expected_capacity(self) -> int: + """one-time property""" + capacity, self._expected_capacity = self._expected_capacity, -1 + return capacity + + def describe_auto_scaling_groups(self, **kwargs: Any) -> dict: + _ = kwargs + return self.expected_data + + def set_desired_capacity(self, **kwargs: Any) -> None: + self._expected_capacity = kwargs["DesiredCapacity"] + + def data_helper( + self, name: str, min_size: int, desired_capacity: int, max_size: int + ) -> None: + self.expected_data = { + "AutoScalingGroups": [ + { + "AutoScalingGroupName": name, + "DesiredCapacity": desired_capacity, + "MinSize": min_size, + "MaxSize": max_size, + } + ] + } + + def setUp(self): + self.client = self.FakeClient() + + def test_normal_cases(self): + test_cases = ( + # Do not change capacity + TestCase("noqueue", 1, 13, 20, [Queue("in_progress", 155, "noqueue")], -1), + TestCase("w/reserve", 1, 13, 20, [Queue("queued", 17, "w/reserve")], -1), + # Increase capacity + TestCase("increase", 1, 13, 20, [Queue("queued", 23, "increase")], 15), + TestCase("increase", 1, 13, 20, [Queue("queued", 18, "increase")], 14), + TestCase("increase", 1, 13, 20, [Queue("queued", 183, "increase")], 20), + TestCase( + "increase-w/o reserve", + 1, + 13, + 20, + [ + Queue("in_progress", 11, "increase-w/o reserve"), + Queue("queued", 12, "increase-w/o reserve"), + ], + 15, + ), + TestCase("lower-min", 10, 5, 20, [Queue("queued", 5, "lower-min")], 10), + # Decrease capacity + TestCase("w/reserve", 1, 13, 20, [Queue("queued", 5, "w/reserve")], 11), + TestCase("w/reserve", 1, 23, 20, [Queue("queued", 17, "w/reserve")], 20), + TestCase("decrease", 1, 13, 20, [Queue("in_progress", 3, "decrease")], 10), + TestCase("decrease", 1, 13, 20, [Queue("in_progress", 5, "decrease")], 11), + ) + for t in test_cases: + self.client.data_helper(t.name, t.min_size, t.desired_capacity, t.max_size) + set_capacity(t.name, t.queues, self.client, False) + self.assertEqual(t.expected_capacity, self.client.expected_capacity, t.name) + + def test_exceptions(self): + test_cases = ( + ( + TestCase( + "different names", + 1, + 1, + 1, + [Queue("queued", 5, "another name")], + -1, + ), + AssertionError, + ), + (TestCase("wrong queue len", 1, 1, 1, [], -1), AssertionError), + ( + TestCase( + "wrong queue", 1, 1, 1, [Queue("wrong", 1, "wrong queue")], -1 # type: ignore + ), + ValueError, + ), + ) + for t, error in test_cases: + with self.assertRaises(error): + self.client.data_helper( + t.name, t.min_size, t.desired_capacity, t.max_size + ) + set_capacity(t.name, t.queues, self.client, False) + + with self.assertRaises(AssertionError): + self.client.expected_data = {"AutoScalingGroups": [1, 2]} + set_capacity( + "wrong number of ASGs", + [Queue("queued", 1, "wrong number of ASGs")], + self.client, + ) diff --git a/tests/ci/ci_config.py b/tests/ci/ci_config.py index 08cd2d466d0..2f35b337cb3 100644 --- a/tests/ci/ci_config.py +++ b/tests/ci/ci_config.py @@ -316,9 +316,12 @@ CI_CONFIG = { "Integration tests flaky check (asan)": { "required_build": "package_asan", }, - "Compatibility check": { + "Compatibility check (amd64)": { "required_build": "package_release", }, + "Compatibility check (aarch64)": { + "required_build": "package_aarch64", + }, "Unit tests (release-clang)": { "required_build": "binary_release", }, diff --git a/tests/ci/compatibility_check.py b/tests/ci/compatibility_check.py index 0bdcf1ba3b4..432e9ec7c01 100644 --- a/tests/ci/compatibility_check.py +++ b/tests/ci/compatibility_check.py @@ -2,6 +2,7 @@ from distutils.version import StrictVersion from typing import List, Tuple +import argparse import logging import os import subprocess @@ -28,9 +29,7 @@ from upload_result_helper import upload_results IMAGE_UBUNTU = "clickhouse/test-old-ubuntu" IMAGE_CENTOS = "clickhouse/test-old-centos" -MAX_GLIBC_VERSION = "2.4" DOWNLOAD_RETRIES_COUNT = 5 -CHECK_NAME = "Compatibility check" def process_os_check(log_path: str) -> TestResult: @@ -43,7 +42,7 @@ def process_os_check(log_path: str) -> TestResult: return TestResult(name, "OK") -def process_glibc_check(log_path: str) -> TestResults: +def process_glibc_check(log_path: str, max_glibc_version: str) -> TestResults: test_results = [] # type: TestResults with open(log_path, "r") as log: for line in log: @@ -53,7 +52,7 @@ def process_glibc_check(log_path: str) -> TestResults: _, version = symbol_with_glibc.split("@GLIBC_") if version == "PRIVATE": test_results.append(TestResult(symbol_with_glibc, "FAIL")) - elif StrictVersion(version) > MAX_GLIBC_VERSION: + elif StrictVersion(version) > max_glibc_version: test_results.append(TestResult(symbol_with_glibc, "FAIL")) if not test_results: test_results.append(TestResult("glibc check", "OK")) @@ -61,18 +60,24 @@ def process_glibc_check(log_path: str) -> TestResults: def process_result( - result_folder: str, server_log_folder: str + result_folder: str, + server_log_folder: str, + check_glibc: bool, + check_distributions: bool, + max_glibc_version: str, ) -> Tuple[str, str, TestResults, List[str]]: glibc_log_path = os.path.join(result_folder, "glibc.log") - test_results = process_glibc_check(glibc_log_path) + test_results = process_glibc_check(glibc_log_path, max_glibc_version) status = "success" description = "Compatibility check passed" - if len(test_results) > 1 or test_results[0].status != "OK": - status = "failure" - description = "glibc check failed" - if status == "success": + if check_glibc: + if len(test_results) > 1 or test_results[0].status != "OK": + status = "failure" + description = "glibc check failed" + + if status == "success" and check_distributions: for operating_system in ("ubuntu:12.04", "centos:5"): test_result = process_os_check( os.path.join(result_folder, operating_system) @@ -101,13 +106,18 @@ def process_result( return status, description, test_results, result_logs -def get_run_commands( - build_path, result_folder, server_log_folder, image_centos, image_ubuntu -): +def get_run_commands_glibc(build_path, result_folder): return [ f"readelf -s --wide {build_path}/usr/bin/clickhouse | grep '@GLIBC_' > {result_folder}/glibc.log", f"readelf -s --wide {build_path}/usr/bin/clickhouse-odbc-bridge | grep '@GLIBC_' >> {result_folder}/glibc.log", f"readelf -s --wide {build_path}/usr/bin/clickhouse-library-bridge | grep '@GLIBC_' >> {result_folder}/glibc.log", + ] + + +def get_run_commands_distributions( + build_path, result_folder, server_log_folder, image_centos, image_ubuntu +): + return [ f"docker run --network=host --volume={build_path}/usr/bin/clickhouse:/clickhouse " f"--volume={build_path}/etc/clickhouse-server:/config " f"--volume={server_log_folder}:/var/log/clickhouse-server {image_ubuntu} > {result_folder}/ubuntu:12.04", @@ -117,9 +127,21 @@ def get_run_commands( ] +def parse_args(): + parser = argparse.ArgumentParser("Check compatibility with old distributions") + parser.add_argument("--check-name", required=True) + parser.add_argument("--check-glibc", action="store_true") + parser.add_argument( + "--check-distributions", action="store_true" + ) # currently hardcoded to x86, don't enable for ARM + return parser.parse_args() + + def main(): logging.basicConfig(level=logging.INFO) + args = parse_args() + stopwatch = Stopwatch() temp_path = TEMP_PATH @@ -129,13 +151,11 @@ def main(): gh = Github(get_best_robot_token(), per_page=100) - rerun_helper = RerunHelper(gh, pr_info, CHECK_NAME) + rerun_helper = RerunHelper(gh, pr_info, args.check_name) if rerun_helper.is_already_finished_by_status(): logging.info("Check is already finished according to github status, exiting") sys.exit(0) - docker_images = get_images_with_versions(reports_path, [IMAGE_CENTOS, IMAGE_UBUNTU]) - packages_path = os.path.join(temp_path, "packages") if not os.path.exists(packages_path): os.makedirs(packages_path) @@ -145,7 +165,7 @@ def main(): "clickhouse-common-static_" in url or "clickhouse-server_" in url ) - download_builds_filter(CHECK_NAME, reports_path, packages_path, url_filter) + download_builds_filter(args.check_name, reports_path, packages_path, url_filter) for f in os.listdir(packages_path): if ".deb" in f: @@ -162,9 +182,24 @@ def main(): if not os.path.exists(result_path): os.makedirs(result_path) - run_commands = get_run_commands( - packages_path, result_path, server_log_path, docker_images[0], docker_images[1] - ) + run_commands = [] + + if args.check_glibc: + check_glibc_commands = get_run_commands_glibc(packages_path, result_path) + run_commands.extend(check_glibc_commands) + + if args.check_distributions: + docker_images = get_images_with_versions( + reports_path, [IMAGE_CENTOS, IMAGE_UBUNTU] + ) + check_distributions_commands = get_run_commands_distributions( + packages_path, + result_path, + server_log_path, + docker_images[0], + docker_images[1], + ) + run_commands.extend(check_distributions_commands) state = "success" for run_command in run_commands: @@ -177,13 +212,26 @@ def main(): subprocess.check_call(f"sudo chown -R ubuntu:ubuntu {temp_path}", shell=True) + # See https://sourceware.org/glibc/wiki/Glibc%20Timeline + max_glibc_version = "" + if "amd64" in args.check_name: + max_glibc_version = "2.4" + elif "aarch64" in args.check_name: + max_glibc_version = "2.18" # because of build with newer sysroot? + else: + raise Exception("Can't determine max glibc version") + s3_helper = S3Helper() state, description, test_results, additional_logs = process_result( - result_path, server_log_path + result_path, + server_log_path, + args.check_glibc, + args.check_distributions, + max_glibc_version, ) ch_helper = ClickHouseHelper() - mark_flaky_tests(ch_helper, CHECK_NAME, test_results) + mark_flaky_tests(ch_helper, args.check_name, test_results) report_url = upload_results( s3_helper, @@ -191,10 +239,10 @@ def main(): pr_info.sha, test_results, additional_logs, - CHECK_NAME, + args.check_name, ) print(f"::notice ::Report url: {report_url}") - post_commit_status(gh, pr_info.sha, CHECK_NAME, description, state, report_url) + post_commit_status(gh, pr_info.sha, args.check_name, description, state, report_url) prepared_events = prepare_tests_results_for_clickhouse( pr_info, @@ -203,7 +251,7 @@ def main(): stopwatch.duration_seconds, stopwatch.start_time_str, report_url, - CHECK_NAME, + args.check_name, ) ch_helper.insert_events_into(db="default", table="checks", events=prepared_events) diff --git a/tests/ci/stress.py b/tests/ci/stress.py index 4f723dba101..5e151e6c098 100755 --- a/tests/ci/stress.py +++ b/tests/ci/stress.py @@ -30,13 +30,16 @@ def get_options(i, upgrade_check): if i % 2 == 1: join_alg_num = i // 2 - if join_alg_num % 4 == 0: + if join_alg_num % 5 == 0: client_options.append("join_algorithm='parallel_hash'") - if join_alg_num % 4 == 1: + if join_alg_num % 5 == 1: client_options.append("join_algorithm='partial_merge'") - if join_alg_num % 4 == 2: + if join_alg_num % 5 == 2: client_options.append("join_algorithm='full_sorting_merge'") - if join_alg_num % 4 == 3: + if join_alg_num % 5 == 3 and not upgrade_check: + # Some crashes are not fixed in 23.2 yet, so ignore the setting in Upgrade check + client_options.append("join_algorithm='grace_hash'") + if join_alg_num % 5 == 4: client_options.append("join_algorithm='auto'") client_options.append("max_rows_in_join=1000") @@ -222,6 +225,20 @@ def prepare_for_hung_check(drop_databases): return True +def is_ubsan_build(): + try: + query = """clickhouse client -q "SELECT value FROM system.build_options WHERE name = 'CXX_FLAGS'" """ + output = ( + check_output(query, shell=True, stderr=STDOUT, timeout=30) + .decode("utf-8") + .strip() + ) + return "-fsanitize=undefined" in output + except Exception as e: + logging.info("Failed to get build flags: %s", str(e)) + return False + + if __name__ == "__main__": logging.basicConfig(level=logging.INFO, format="%(asctime)s %(message)s") parser = argparse.ArgumentParser( @@ -241,6 +258,10 @@ if __name__ == "__main__": args = parser.parse_args() if args.drop_databases and not args.hung_check: raise Exception("--drop-databases only used in hung check (--hung-check)") + + # FIXME Hung check with ubsan is temporarily disabled due to https://github.com/ClickHouse/ClickHouse/issues/45372 + suppress_hung_check = is_ubsan_build() + func_pipes = [] func_pipes = run_func_test( args.test_cmd, @@ -305,7 +326,7 @@ if __name__ == "__main__": res = call(cmd, shell=True, stdout=tee.stdin, stderr=STDOUT) if tee.stdin is not None: tee.stdin.close() - if res != 0 and have_long_running_queries: + if res != 0 and have_long_running_queries and not suppress_hung_check: logging.info("Hung check failed with exit code %d", res) else: hung_check_status = "No queries hung\tOK\t\\N\t\n" diff --git a/tests/ci/workflow_jobs_lambda/app.py b/tests/ci/workflow_jobs_lambda/app.py index f6589576806..9436e01ad53 100644 --- a/tests/ci/workflow_jobs_lambda/app.py +++ b/tests/ci/workflow_jobs_lambda/app.py @@ -213,7 +213,7 @@ def send_event_workflow_job(workflow_job: WorkflowJob) -> None: # `head_sha` String, # `url` String, # `html_url` String, - # `status` Enum8('queued' = 1, 'in_progress' = 2, 'completed' = 3, 'waiting' = 4), + # `status` Enum8('waiting' = 1, 'queued' = 2, 'in_progress' = 3, 'completed' = 4), # `conclusion` LowCardinality(String), # `started_at` DateTime, # `completed_at` DateTime, diff --git a/tests/clickhouse-test b/tests/clickhouse-test index 68bca93c55c..aec52981724 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -76,8 +76,9 @@ def trim_for_log(s): if not s: return s lines = s.splitlines() - if len(lines) > 100: - return "\n".join(lines[:50] + ["#" * 100] + lines[-50:]) + if len(lines) > 10000: + separator = "-" * 40 + str(len(lines) - 10000) + " lines are hidden" + "-" * 40 + return "\n".join(lines[:5000] + [] + [separator] + [] + lines[-5000:]) else: return "\n".join(lines) @@ -94,7 +95,7 @@ class HTTPError(Exception): # Helpers to execute queries via HTTP interface. def clickhouse_execute_http( - base_args, query, timeout=30, settings=None, default_format=None, max_http_retries=5 + base_args, query, timeout=30, settings=None, default_format=None, max_http_retries=5, retry_error_codes=False ): if args.secure: client = http.client.HTTPSConnection( @@ -117,6 +118,7 @@ def clickhouse_execute_http( "http_connection_timeout": timeout, "http_receive_timeout": timeout, "http_send_timeout": timeout, + "output_format_parallel_formatting": 0, } if settings is not None: params.update(settings) @@ -131,7 +133,8 @@ def clickhouse_execute_http( ) res = client.getresponse() data = res.read() - break + if res.status == 200 or (not retry_error_codes): + break except Exception as ex: if i == max_http_retries - 1: raise ex @@ -143,8 +146,8 @@ def clickhouse_execute_http( return data -def clickhouse_execute(base_args, query, timeout=30, settings=None, max_http_retries=5): - return clickhouse_execute_http(base_args, query, timeout, settings, max_http_retries=max_http_retries).strip() +def clickhouse_execute(base_args, query, timeout=30, settings=None, max_http_retries=5, retry_error_codes=False): + return clickhouse_execute_http(base_args, query, timeout, settings, max_http_retries=max_http_retries, retry_error_codes=retry_error_codes).strip() def clickhouse_execute_json(base_args, query, timeout=60, settings=None, max_http_retries=5): @@ -434,7 +437,7 @@ class FailureReason(enum.Enum): SERVER_DIED = "server died" EXIT_CODE = "return code: " STDERR = "having stderror: " - EXCEPTION = "having having exception in stdout: " + EXCEPTION = "having exception in stdout: " RESULT_DIFF = "result differs with reference: " TOO_LONG = "Test runs too long (> 60s). Make it faster." INTERNAL_QUERY_FAIL = "Internal query (CREATE/DROP DATABASE) failed:" @@ -520,7 +523,11 @@ class SettingsRandomizer: "merge_tree_coarse_index_granularity": lambda: random.randint(2, 32), "optimize_distinct_in_order": lambda: random.randint(0, 1), "optimize_sorting_by_input_stream_properties": lambda: random.randint(0, 1), - "enable_memory_bound_merging_of_aggregation_results": lambda: random.randint(0, 1), + "http_response_buffer_size": lambda: random.randint(0, 10 * 1048576), + "http_wait_end_of_query": lambda: random.random() > 0.5, + "enable_memory_bound_merging_of_aggregation_results": lambda: random.randint( + 0, 1 + ), } @staticmethod @@ -533,10 +540,9 @@ class SettingsRandomizer: class MergeTreeSettingsRandomizer: settings = { - # Temporary disable due to large number of failures. TODO: fix. - # "ratio_of_defaults_for_sparse_serialization": threshold_generator( - # 0.1, 0.6, 0.0, 1.0 - # ), + "ratio_of_defaults_for_sparse_serialization": threshold_generator( + 0.3, 0.5, 0.0, 1.0 + ), "prefer_fetch_merged_part_size_threshold": threshold_generator( 0.2, 0.5, 1, 10 * 1024 * 1024 * 1024 ), @@ -1045,7 +1051,7 @@ class TestCase: @staticmethod def send_test_name_failed(suite: str, case: str): pid = os.getpid() - clickhouse_execute(args, f"SELECT 'Running test {suite}/{case} from pid={pid}'") + clickhouse_execute(args, f"SELECT 'Running test {suite}/{case} from pid={pid}'", retry_error_codes=True) def run_single_test( self, server_logs_level, client_options @@ -1202,7 +1208,7 @@ class TestCase: ) result.check_if_need_retry(args, stdout, stderr, self.runs_count) # to avoid breaking CSV parser - result.description = result.description.replace('\0', '') + result.description = result.description.replace("\0", "") if result.status == TestStatus.FAIL: result.description = self.add_info_about_settings(result.description) @@ -1658,7 +1664,12 @@ def check_server_started(args): print("\nConnection timeout, will not retry") break except Exception as e: - print("\nUexpected exception, will not retry: ", type(e).__name__, ": ", str(e)) + print( + "\nUexpected exception, will not retry: ", + type(e).__name__, + ": ", + str(e), + ) break print("\nAll connection tries failed") @@ -1935,7 +1946,10 @@ def reportCoverage(args): ) ) + def reportLogStats(args): + clickhouse_execute(args, "SYSTEM FLUSH LOGS") + query = """ WITH 120 AS mins, @@ -2004,7 +2018,13 @@ def reportLogStats(args): 'Cyclic aliases', 'Detaching {}', 'Executing {}', 'Fire events: {}', 'Found part {}', 'Loaded queue', 'No sharding key', 'No tables', 'Query: {}', 'Removed', 'Removed part {}', 'Removing parts.', 'Request URI: {}', 'Sending part {}', 'Sent handshake', 'Starting {}', 'Will mimic {}', 'Writing to {}', - 'dropIfEmpty', 'loadAll {}', '{} ({}:{})', '{} -> {}', '{} {}', '{}: {}' + 'dropIfEmpty', 'loadAll {}', '{} ({}:{})', '{} -> {}', '{} {}', '{}: {}', 'Query was cancelled', + 'Table {} already exists', '{}%', 'Cancelled merging parts', 'All replicas are lost', + 'Cancelled mutating parts', 'Read object: {}', 'New segment: {}', 'Unknown geometry type {}', + 'Table {} is not replicated', '{} {}.{} already exists', 'Attempt to read after eof', + 'Replica {} already exists', 'Convert overflow', 'key must be a tuple', 'Division by zero', + 'No part {} in committed state', 'Files set to {}', 'Bytes set to {}', 'Sharding key {} is not used', + 'Cannot parse datetime', 'Bad get: has {}, requested {}', 'There is no {} in {}', 'Numeric overflow' ) AS known_short_messages SELECT count() AS c, message_format_string, substr(any(message), 1, 120) FROM system.text_log @@ -2228,7 +2248,8 @@ def parse_args(): parser.add_argument( "-b", "--binary", - default=find_binary("clickhouse"), + default="clickhouse", + type=find_binary, help="Path to clickhouse binary or name of binary in PATH", ) parser.add_argument( diff --git a/tests/config/config.d/keeper_port.xml b/tests/config/config.d/keeper_port.xml index 2066dedfa56..cffd325e968 100644 --- a/tests/config/config.d/keeper_port.xml +++ b/tests/config/config.d/keeper_port.xml @@ -9,8 +9,7 @@ 10000 100000 10000 - - true + false 240000 1000000000000000 diff --git a/tests/integration/test_async_drain_connection/configs/config.xml b/tests/integration/test_async_drain_connection/configs/config.xml deleted file mode 100644 index 88862a3001e..00000000000 --- a/tests/integration/test_async_drain_connection/configs/config.xml +++ /dev/null @@ -1,3 +0,0 @@ - - 10000 - diff --git a/tests/integration/test_async_drain_connection/test.py b/tests/integration/test_async_drain_connection/test.py deleted file mode 100644 index 66786f4a8f9..00000000000 --- a/tests/integration/test_async_drain_connection/test.py +++ /dev/null @@ -1,41 +0,0 @@ -# pylint: disable=redefined-outer-name -# pylint: disable=unused-argument - -import pytest -from helpers.cluster import ClickHouseCluster - -cluster = ClickHouseCluster(__file__) -node = cluster.add_instance("node", main_configs=["configs/config.xml"]) - - -@pytest.fixture(scope="module") -def started_cluster(): - try: - cluster.start() - node.query( - """ - create table t (number UInt64) - engine = Distributed(test_cluster_two_shards, system, numbers) - """ - ) - yield cluster - - finally: - cluster.shutdown() - - -def test_filled_async_drain_connection_pool(started_cluster): - def execute_queries(_): - for _ in range(100): - node.query( - "select * from t where number = 0 limit 2", - settings={ - "sleep_in_receive_cancel_ms": int(10e6), - "max_execution_time": 5, - # decrease drain_timeout to make test more stable - # (another way is to increase max_execution_time, but this will make test slower) - "drain_timeout": 1, - }, - ) - - any(map(execute_queries, range(10))) diff --git a/tests/integration/test_async_insert_memory/test.py b/tests/integration/test_async_insert_memory/test.py deleted file mode 100644 index 279542f087c..00000000000 --- a/tests/integration/test_async_insert_memory/test.py +++ /dev/null @@ -1,40 +0,0 @@ -import pytest - -from helpers.cluster import ClickHouseCluster - -cluster = ClickHouseCluster(__file__) - -node = cluster.add_instance("node") - - -@pytest.fixture(scope="module", autouse=True) -def start_cluster(): - try: - cluster.start() - yield cluster - finally: - cluster.shutdown() - - -def test_memory_usage(): - node.query( - "CREATE TABLE async_table(data Array(UInt64)) ENGINE=MergeTree() ORDER BY data" - ) - - node.get_query_request("SELECT count() FROM system.numbers") - - INSERT_QUERY = "INSERT INTO async_table SETTINGS async_insert=1, wait_for_async_insert=1 VALUES ({})" - for iter in range(10): - values = list(range(iter * 5000000, (iter + 1) * 5000000)) - node.query(INSERT_QUERY.format(values)) - - response = node.get_query_request( - "SELECT groupArray(number) FROM numbers(1000000) SETTINGS max_memory_usage_for_user={}".format( - 30 * (2**23) - ) - ) - - _, err = response.get_answer_and_error() - assert err == "", "Query failed with error {}".format(err) - - node.query("DROP TABLE async_table") diff --git a/tests/integration/test_backup_restore_s3/test.py b/tests/integration/test_backup_restore_s3/test.py index d5a7579df51..d046c97f8d1 100644 --- a/tests/integration/test_backup_restore_s3/test.py +++ b/tests/integration/test_backup_restore_s3/test.py @@ -138,7 +138,8 @@ def test_backup_to_s3_native_copy(): f"S3('http://minio1:9001/root/data/backups/{backup_name}', 'minio', 'minio123')" ) check_backup_and_restore(storage_policy, backup_destination) - assert node.contains_in_log("using native copy") + assert node.contains_in_log("BackupImpl.*using native copy") + assert node.contains_in_log("copyS3FileToDisk.*using native copy") assert node.contains_in_log( f"copyS3File: Single operation copy has completed. Bucket: root, Key: data/backups/{backup_name}" ) @@ -151,7 +152,8 @@ def test_backup_to_s3_native_copy_other_bucket(): f"S3('http://minio1:9001/root/data/backups/{backup_name}', 'minio', 'minio123')" ) check_backup_and_restore(storage_policy, backup_destination) - assert node.contains_in_log("using native copy") + assert node.contains_in_log("BackupImpl.*using native copy") + assert node.contains_in_log("copyS3FileToDisk.*using native copy") assert node.contains_in_log( f"copyS3File: Single operation copy has completed. Bucket: root, Key: data/backups/{backup_name}" ) @@ -162,7 +164,8 @@ def test_backup_to_s3_native_copy_multipart(): backup_name = new_backup_name() backup_destination = f"S3('http://minio1:9001/root/data/backups/multipart/{backup_name}', 'minio', 'minio123')" check_backup_and_restore(storage_policy, backup_destination, size=1000000) - assert node.contains_in_log("using native copy") + assert node.contains_in_log("BackupImpl.*using native copy") + assert node.contains_in_log("copyS3FileToDisk.*using native copy") assert node.contains_in_log( f"copyS3File: Multipart upload has completed. Bucket: root, Key: data/backups/multipart/{backup_name}/" ) diff --git a/tests/integration/test_backward_compatibility/test_convert_ordinary.py b/tests/integration/test_backward_compatibility/test_convert_ordinary.py index afc44c91fc4..8b1afd358eb 100644 --- a/tests/integration/test_backward_compatibility/test_convert_ordinary.py +++ b/tests/integration/test_backward_compatibility/test_convert_ordinary.py @@ -188,7 +188,18 @@ def check_convert_all_dbs_to_atomic(): node.exec_in_container( ["bash", "-c", f"touch /var/lib/clickhouse/flags/convert_ordinary_to_atomic"] ) - node.restart_clickhouse() + node.stop_clickhouse() + cannot_start = False + try: + node.start_clickhouse() + except: + cannot_start = True + assert cannot_start + + node.exec_in_container( + ["bash", "-c", f"rm /var/lib/clickhouse/flags/convert_ordinary_to_atomic"] + ) + node.start_clickhouse() assert "Ordinary\n" == node.query( "SELECT engine FROM system.databases where name='ordinary'" diff --git a/tests/integration/test_disk_configuration/test.py b/tests/integration/test_disk_configuration/test.py index 34f8bea219f..6ebe994dc68 100644 --- a/tests/integration/test_disk_configuration/test.py +++ b/tests/integration/test_disk_configuration/test.py @@ -294,6 +294,65 @@ def test_merge_tree_custom_disk_setting(start_cluster): ).strip() ) + node1.query(f"DROP TABLE {TABLE_NAME} SYNC") + node1.query(f"DROP TABLE {TABLE_NAME}_2 SYNC") + node1.query(f"DROP TABLE {TABLE_NAME}_3 SYNC") + node1.query(f"DROP TABLE {TABLE_NAME}_4 SYNC") + node2.query(f"DROP TABLE {TABLE_NAME}_4 SYNC") + + +def test_merge_tree_nested_custom_disk_setting(start_cluster): + node = cluster.instances["node1"] + + minio = cluster.minio_client + for obj in list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True)): + minio.remove_object(cluster.minio_bucket, obj.object_name) + assert ( + len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) + == 0 + ) + + node.query( + f""" + DROP TABLE IF EXISTS {TABLE_NAME} SYNC; + CREATE TABLE {TABLE_NAME} (a Int32) + ENGINE = MergeTree() order by tuple() + SETTINGS disk = disk( + type=cache, + max_size='1Gi', + path='/var/lib/clickhouse/custom_disk_cache/', + disk=disk( + type=s3, + endpoint='http://minio1:9001/root/data/', + access_key_id='minio', + secret_access_key='minio123')); + """ + ) + + node.query(f"INSERT INTO {TABLE_NAME} SELECT number FROM numbers(100)") + node.query("SYSTEM DROP FILESYSTEM CACHE") + + # Check cache is filled + assert 0 == int(node.query("SELECT count() FROM system.filesystem_cache")) + assert 100 == int(node.query(f"SELECT count() FROM {TABLE_NAME}")) + node.query(f"SELECT * FROM {TABLE_NAME}") + assert 0 < int(node.query("SELECT count() FROM system.filesystem_cache")) + + # Check s3 is filled + assert ( + len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) > 0 + ) + + node.restart_clickhouse() + + assert 100 == int(node.query(f"SELECT count() FROM {TABLE_NAME}")) + + expected = """ + SETTINGS disk = disk(type = cache, max_size = \\'[HIDDEN]\\', path = \\'[HIDDEN]\\', disk = disk(type = s3, endpoint = \\'[HIDDEN]\\' + """ + assert expected.strip() in node.query(f"SHOW CREATE TABLE {TABLE_NAME}").strip() + node.query(f"DROP TABLE {TABLE_NAME} SYNC") + def test_merge_tree_setting_override(start_cluster): node = cluster.instances["node3"] @@ -367,3 +426,4 @@ def test_merge_tree_setting_override(start_cluster): assert ( len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) > 0 ) + node.query(f"DROP TABLE {TABLE_NAME} SYNC") diff --git a/tests/integration/test_distributed_directory_monitor_split_batch_on_failure/configs/overrides_1.xml b/tests/integration/test_distributed_directory_monitor_split_batch_on_failure/configs/overrides_1.xml index 397e05e7a60..a79ce3de1fc 100644 --- a/tests/integration/test_distributed_directory_monitor_split_batch_on_failure/configs/overrides_1.xml +++ b/tests/integration/test_distributed_directory_monitor_split_batch_on_failure/configs/overrides_1.xml @@ -3,8 +3,6 @@ 0 - - 1 1 diff --git a/tests/integration/test_distributed_directory_monitor_split_batch_on_failure/configs/overrides_2.xml b/tests/integration/test_distributed_directory_monitor_split_batch_on_failure/configs/overrides_2.xml index 2ffd5beaf8d..8279fcdbe6d 100644 --- a/tests/integration/test_distributed_directory_monitor_split_batch_on_failure/configs/overrides_2.xml +++ b/tests/integration/test_distributed_directory_monitor_split_batch_on_failure/configs/overrides_2.xml @@ -3,8 +3,6 @@ 0 - - 1 0 diff --git a/tests/integration/test_distributed_directory_monitor_split_batch_on_failure/test.py b/tests/integration/test_distributed_directory_monitor_split_batch_on_failure/test.py index a47268b06fd..faa38af6533 100644 --- a/tests/integration/test_distributed_directory_monitor_split_batch_on_failure/test.py +++ b/tests/integration/test_distributed_directory_monitor_split_batch_on_failure/test.py @@ -18,61 +18,86 @@ node2 = cluster.add_instance( ) +def get_test_settings(): + settings = {"monitor_batch_inserts": [0, 1]} + return [(k, v) for k, values in settings.items() for v in values] + + +def drop_tables(): + tables = ["null_", "dist", "data", "mv", "dist_data"] + query = "\n".join([f"drop table if exists {table};" for table in tables]) + for _, node in cluster.instances.items(): + node.query(query) + + +def create_tables(**dist_settings): + drop_tables() + _settings_values = ",".join([f"{k}={v}" for k, v in dist_settings.items()]) + _settings = f"settings {_settings_values}" if _settings_values else "" + for _, node in cluster.instances.items(): + node.query( + f""" + create table null_ (key Int, value Int) engine=Null(); + create table dist as null_ engine=Distributed(test_cluster, currentDatabase(), null_, key) {_settings}; + create table data (key Int, uniq_values Int) engine=Memory(); + create materialized view mv to data as select key, uniqExact(value) uniq_values from null_ group by key; + system stop distributed sends dist; + + create table dist_data as data engine=Distributed(test_cluster, currentDatabase(), data); + """ + ) + + @pytest.fixture(scope="module") def started_cluster(): try: cluster.start() - - for _, node in cluster.instances.items(): - node.query( - """ - create table null_ (key Int, value Int) engine=Null(); - create table dist as null_ engine=Distributed(test_cluster, currentDatabase(), null_, key); - create table data (key Int, uniq_values Int) engine=Memory(); - create materialized view mv to data as select key, uniqExact(value) uniq_values from null_ group by key; - system stop distributed sends dist; - - create table dist_data as data engine=Distributed(test_cluster, currentDatabase(), data); - """ - ) - yield cluster finally: + drop_tables() cluster.shutdown() def test_distributed_directory_monitor_split_batch_on_failure_OFF(started_cluster): - for i in range(0, 100): - limit = 100e3 - node2.query( - f"insert into dist select number/100, number from system.numbers limit {limit} offset {limit*i}", - settings={ - # max_memory_usage is the limit for the batch on the remote node - # (local query should not be affected since 30MB is enough for 100K rows) - "max_memory_usage": "30Mi", - "max_untracked_memory": "0", - }, - ) - # "Received from" is mandatory, since the exception should be thrown on the remote node. - with pytest.raises( - QueryRuntimeException, - match=r"DB::Exception: Received from.*Memory limit \(for query\) exceeded: .*while pushing to view default\.mv", - ): + for setting, setting_value in get_test_settings(): + create_tables(**{setting: setting_value}) + for i in range(0, 100): + limit = 100e3 + node2.query( + f"insert into dist select number/100, number from system.numbers limit {limit} offset {limit*i}", + settings={ + # max_memory_usage is the limit for the batch on the remote node + # (local query should not be affected since 30MB is enough for 100K rows) + "max_memory_usage": "30Mi", + "max_untracked_memory": "0", + }, + ) + # "Received from" is mandatory, since the exception should be thrown on the remote node. + if setting == "monitor_batch_inserts" and setting_value == 1: + with pytest.raises( + QueryRuntimeException, + match=r"DB::Exception: Received from.*Memory limit \(for query\) exceeded: .*while pushing to view default\.mv", + ): + node2.query("system flush distributed dist") + assert int(node2.query("select count() from dist_data")) == 0 + continue node2.query("system flush distributed dist") - assert int(node2.query("select count() from dist_data")) == 0 + assert int(node2.query("select count() from dist_data")) == 100000 def test_distributed_directory_monitor_split_batch_on_failure_ON(started_cluster): - for i in range(0, 100): - limit = 100e3 - node1.query( - f"insert into dist select number/100, number from system.numbers limit {limit} offset {limit*i}", - settings={ - # max_memory_usage is the limit for the batch on the remote node - # (local query should not be affected since 30MB is enough for 100K rows) - "max_memory_usage": "30Mi", - "max_untracked_memory": "0", - }, - ) - node1.query("system flush distributed dist") - assert int(node1.query("select count() from dist_data")) == 100000 + for setting, setting_value in get_test_settings(): + create_tables(**{setting: setting_value}) + for i in range(0, 100): + limit = 100e3 + node1.query( + f"insert into dist select number/100, number from system.numbers limit {limit} offset {limit*i}", + settings={ + # max_memory_usage is the limit for the batch on the remote node + # (local query should not be affected since 30MB is enough for 100K rows) + "max_memory_usage": "30Mi", + "max_untracked_memory": "0", + }, + ) + node1.query("system flush distributed dist") + assert int(node1.query("select count() from dist_data")) == 100000 diff --git a/tests/integration/test_distributed_inter_server_secret/configs/remote_servers.xml b/tests/integration/test_distributed_inter_server_secret/configs/remote_servers.xml index f5888a974f1..163850c6f2d 100644 --- a/tests/integration/test_distributed_inter_server_secret/configs/remote_servers.xml +++ b/tests/integration/test_distributed_inter_server_secret/configs/remote_servers.xml @@ -24,5 +24,17 @@ 1 + + + foo_backward + + n1 + 9000 + + + backward + 9000 + + diff --git a/tests/integration/test_distributed_inter_server_secret/configs/remote_servers_backward.xml b/tests/integration/test_distributed_inter_server_secret/configs/remote_servers_backward.xml new file mode 100644 index 00000000000..3636b402145 --- /dev/null +++ b/tests/integration/test_distributed_inter_server_secret/configs/remote_servers_backward.xml @@ -0,0 +1,15 @@ + + + + backward + + n1 + 9000 + + + backward + 9000 + + + + diff --git a/tests/integration/test_distributed_inter_server_secret/test.py b/tests/integration/test_distributed_inter_server_secret/test.py index 4f1c4b9b749..6dd25789f36 100644 --- a/tests/integration/test_distributed_inter_server_secret/test.py +++ b/tests/integration/test_distributed_inter_server_secret/test.py @@ -12,18 +12,28 @@ from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -def make_instance(name, cfg): +def make_instance(name, cfg, *args, **kwargs): return cluster.add_instance( name, with_zookeeper=True, main_configs=["configs/remote_servers.xml", cfg], user_configs=["configs/users.xml"], + *args, + **kwargs, ) # _n1/_n2 contains cluster with different -- should fail n1 = make_instance("n1", "configs/remote_servers_n1.xml") n2 = make_instance("n2", "configs/remote_servers_n2.xml") +backward = make_instance( + "backward", + "configs/remote_servers_backward.xml", + image="clickhouse/clickhouse-server", + # version without DBMS_MIN_REVISION_WITH_INTERSERVER_SECRET_V2 + tag="23.2.3", + with_installed_binary=True, +) users = pytest.mark.parametrize( "user,password", @@ -54,6 +64,12 @@ def bootstrap(): Engine=Distributed(secure, currentDatabase(), data, key) """ ) + n.query( + """ + CREATE TABLE dist_secure_backward AS data + Engine=Distributed(secure_backward, currentDatabase(), data, key) + """ + ) n.query( """ CREATE TABLE dist_secure_from_buffer AS data_from_buffer @@ -409,3 +425,31 @@ def test_per_user_protocol_settings_secure_cluster(user, password): assert int(get_query_setting_on_shard(n1, id_, "max_memory_usage_for_user")) == int( 1e9 ) + + +@users +def test_user_secure_cluster_with_backward(user, password): + id_ = "with-backward-query-dist_secure-" + user + query_with_id( + n1, id_, "SELECT * FROM dist_secure_backward", user=user, password=password + ) + assert get_query_user_info(n1, id_) == [user, user] + assert get_query_user_info(backward, id_) == [user, user] + + +@users +def test_user_secure_cluster_from_backward(user, password): + id_ = "from-backward-query-dist_secure-" + user + query_with_id( + backward, + id_, + "SELECT * FROM dist_secure_backward", + user=user, + password=password, + ) + assert get_query_user_info(n1, id_) == [user, user] + assert get_query_user_info(backward, id_) == [user, user] + + assert n1.contains_in_log( + "Using deprecated interserver protocol because the client is too old. Consider upgrading all nodes in cluster." + ) diff --git a/tests/integration/test_filesystem_layout/test.py b/tests/integration/test_filesystem_layout/test.py index 898bbc40eb9..2be478f95d0 100644 --- a/tests/integration/test_filesystem_layout/test.py +++ b/tests/integration/test_filesystem_layout/test.py @@ -44,8 +44,6 @@ def test_file_path_escaping(started_cluster): ] ) - -def test_file_path_escaping_atomic_db(started_cluster): node.query("CREATE DATABASE IF NOT EXISTS `test 2` ENGINE = Atomic") node.query( """ diff --git a/tests/integration/test_grpc_protocol/test.py b/tests/integration/test_grpc_protocol/test.py index a1bc0d42a46..137d585f7d1 100644 --- a/tests/integration/test_grpc_protocol/test.py +++ b/tests/integration/test_grpc_protocol/test.py @@ -594,8 +594,6 @@ def test_cancel_while_processing_input(): stub = clickhouse_grpc_pb2_grpc.ClickHouseStub(main_channel) result = stub.ExecuteQueryWithStreamInput(send_query_info()) assert result.cancelled == True - assert result.progress.written_rows == 6 - assert query("SELECT a FROM t ORDER BY a") == "1\n2\n3\n4\n5\n6\n" def test_cancel_while_generating_output(): diff --git a/tests/integration/test_http_handlers_config/test.py b/tests/integration/test_http_handlers_config/test.py index 9c64bd41b23..1b347f6271f 100644 --- a/tests/integration/test_http_handlers_config/test.py +++ b/tests/integration/test_http_handlers_config/test.py @@ -147,6 +147,18 @@ def test_predefined_query_handler(): assert b"max_final_threads\t1\nmax_threads\t1\n" == res2.content assert "application/generic+one" == res2.headers["content-type"] + cluster.instance.query( + "CREATE TABLE test_table (id UInt32, data String) Engine=TinyLog" + ) + res3 = cluster.instance.http_request( + "test_predefined_handler_post_body?id=100", + method="POST", + data="TEST".encode("utf8"), + ) + assert res3.status_code == 200 + assert cluster.instance.query("SELECT * FROM test_table") == "100\tTEST\n" + cluster.instance.query("DROP TABLE test_table") + def test_fixed_static_handler(): with contextlib.closing( diff --git a/tests/integration/test_http_handlers_config/test_predefined_handler/config.xml b/tests/integration/test_http_handlers_config/test_predefined_handler/config.xml index 5b5de63356e..1b8ddfab323 100644 --- a/tests/integration/test_http_handlers_config/test_predefined_handler/config.xml +++ b/tests/integration/test_http_handlers_config/test_predefined_handler/config.xml @@ -21,5 +21,13 @@ application/generic+one + + POST + /test_predefined_handler_post_body + + predefined_query_handler + INSERT INTO test_table(id, data) SELECT {id:UInt32}, {_request_body:String} + + diff --git a/tests/integration/test_insert_into_distributed/test.py b/tests/integration/test_insert_into_distributed/test.py index a52809f817c..3bee2149387 100644 --- a/tests/integration/test_insert_into_distributed/test.py +++ b/tests/integration/test_insert_into_distributed/test.py @@ -288,7 +288,7 @@ def test_inserts_single_replica_no_internal_replication(started_cluster): "prefer_localhost_replica": "0", }, ) - assert node2.query("SELECT count(*) FROM single_replicated").strip() == "1" + assert node2.query("SELECT count(*) FROM single_replicated").strip() == "0" finally: node2.query("TRUNCATE TABLE single_replicated") diff --git a/tests/integration/test_keeper_four_word_command/test.py b/tests/integration/test_keeper_four_word_command/test.py index 04f6800b92b..412780c8f0f 100644 --- a/tests/integration/test_keeper_four_word_command/test.py +++ b/tests/integration/test_keeper_four_word_command/test.py @@ -679,3 +679,44 @@ def test_cmd_rqld(started_cluster): + " does not become leader after 30s, maybe there is something wrong." ) assert keeper_utils.is_leader(cluster, node) + + +def test_cmd_clrs(started_cluster): + if node1.is_built_with_sanitizer(): + return + + def get_memory_purges(): + return node1.query( + "SELECT value FROM system.events WHERE event = 'MemoryAllocatorPurge' SETTINGS system_events_show_zero_values = 1" + ) + + zk = None + try: + wait_nodes() + + zk = get_fake_zk(node1.name, timeout=30.0) + + paths = [f"/clrs_{i}" for i in range(10000)] + + # we only count the events because we cannot reliably test memory usage of Keeper + # but let's create and delete nodes so the first purge needs to release some memory + create_transaction = zk.transaction() + for path in paths: + create_transaction.create(path) + create_transaction.commit() + + delete_transaction = zk.transaction() + for path in paths: + delete_transaction.delete(path) + delete_transaction.commit() + + # repeat multiple times to make sure MemoryAllocatorPurge isn't increased because of other reasons + for _ in range(5): + prev_purges = int(get_memory_purges()) + keeper_utils.send_4lw_cmd(cluster, node1, cmd="clrs") + current_purges = int(get_memory_purges()) + assert current_purges > prev_purges + prev_purges = current_purges + + finally: + destroy_zk_client(zk) diff --git a/tests/integration/test_async_drain_connection/__init__.py b/tests/integration/test_keeper_mntr_data_size/__init__.py similarity index 100% rename from tests/integration/test_async_drain_connection/__init__.py rename to tests/integration/test_keeper_mntr_data_size/__init__.py diff --git a/tests/integration/test_keeper_mntr_data_size/configs/enable_keeper.xml b/tests/integration/test_keeper_mntr_data_size/configs/enable_keeper.xml new file mode 100644 index 00000000000..a3217b34501 --- /dev/null +++ b/tests/integration/test_keeper_mntr_data_size/configs/enable_keeper.xml @@ -0,0 +1,29 @@ + + + 9181 + 1 + /var/lib/clickhouse/coordination/log + /var/lib/clickhouse/coordination/snapshots + + + 10 + 5 + 5000 + 10000 + trace + + + 0 + 0 + 0 + + + + + 1 + localhost + 9234 + + + + diff --git a/tests/integration/test_keeper_mntr_data_size/test.py b/tests/integration/test_keeper_mntr_data_size/test.py new file mode 100644 index 00000000000..8789ca0354c --- /dev/null +++ b/tests/integration/test_keeper_mntr_data_size/test.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 + +import pytest +from helpers.cluster import ClickHouseCluster +import helpers.keeper_utils as keeper_utils +import random +import string +import os +import time +from kazoo.client import KazooClient, KazooState + + +cluster = ClickHouseCluster(__file__) + +# clickhouse itself will use external zookeeper +node = cluster.add_instance( + "node", + main_configs=["configs/enable_keeper.xml"], + stay_alive=True, + with_zookeeper=True, +) + + +def random_string(length): + return "".join(random.choices(string.ascii_lowercase + string.digits, k=length)) + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + + yield cluster + + finally: + cluster.shutdown() + + +def get_connection_zk(nodename, timeout=30.0): + _fake_zk_instance = KazooClient( + hosts=cluster.get_instance_ip(nodename) + ":9181", timeout=timeout + ) + _fake_zk_instance.start() + return _fake_zk_instance + + +def restart_clickhouse(): + node.restart_clickhouse() + keeper_utils.wait_until_connected(cluster, node) + + +def test_mntr_data_size_after_restart(started_cluster): + try: + node_zk = None + node_zk = get_connection_zk("node") + + node_zk.create("/test_mntr_data_size", b"somevalue") + for i in range(100): + node_zk.create( + "/test_mntr_data_size/node" + str(i), random_string(123).encode() + ) + + def get_line_with_size(): + return next( + filter( + lambda line: "zk_approximate_data_size" in line, + keeper_utils.send_4lw_cmd(started_cluster, node, "mntr").split( + "\n" + ), + ), + None, + ) + + line_size_before = get_line_with_size() + assert line_size_before != None + + node_zk.stop() + node_zk.close() + node_zk = None + + restart_clickhouse() + + assert get_line_with_size() == line_size_before + + keeper_utils.send_4lw_cmd(started_cluster, node, "rclc") + assert get_line_with_size() == line_size_before + finally: + try: + if node_zk is not None: + node_zk.stop() + node_zk.close() + except: + pass diff --git a/tests/integration/test_log_levels_update/test.py b/tests/integration/test_log_levels_update/test.py index 176733cd7cb..4b83b6431fc 100644 --- a/tests/integration/test_log_levels_update/test.py +++ b/tests/integration/test_log_levels_update/test.py @@ -10,7 +10,7 @@ node = cluster.add_instance( config = """ - information + debug /var/log/clickhouse-server/clickhouse-server.log """ @@ -63,4 +63,4 @@ def test_log_levels_update(start_cluster): log = get_log(node) assert len(log) > 0 - assert not re.search("(|)", log) + assert not re.search("", log) diff --git a/tests/integration/test_materialized_mysql_database/materialize_with_ddl.py b/tests/integration/test_materialized_mysql_database/materialize_with_ddl.py index 5b75b0dfc38..97e2de49ceb 100644 --- a/tests/integration/test_materialized_mysql_database/materialize_with_ddl.py +++ b/tests/integration/test_materialized_mysql_database/materialize_with_ddl.py @@ -2063,7 +2063,7 @@ def materialized_database_support_all_kinds_of_mysql_datatype( # increment synchronization check check_query( clickhouse_node, - "SELECT v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, hex(v25), v26, v28, v29, v30, v32 FROM test_database_datatype.t1 FORMAT TSV", + "SELECT v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, hex(v25), v26, v28, v29, v30, v32 FROM test_database_datatype.t1 ORDER BY v1 FORMAT TSV", "1\t1\t11\t9223372036854775807\t-1\t1\t11\t18446744073709551615\t-1.1\t1.1\t-1.111\t1.111\t1.1111\t2021-10-06\ttext\tvarchar\tBLOB\t2021-10-06 18:32:57\t2021-10-06 18:32:57.482786\t2021-10-06 18:32:57\t2021-10-06 18:32:57.482786" + "\t2021\t3020399000000\t3020399000000\t00000000010100000000000000000000000000000000000000\t10\t1\t11\tvarbinary\tRED\n" + "2\t2\t22\t9223372036854775807\t-2\t2\t22\t18446744073709551615\t-2.2\t2.2\t-2.22\t2.222\t2.2222\t2021-10-07\ttext\tvarchar\tBLOB\t2021-10-07 18:32:57\t2021-10-07 18:32:57.482786\t2021-10-07 18:32:57\t2021-10-07 18:32:57.482786" diff --git a/tests/integration/test_async_insert_memory/__init__.py b/tests/integration/test_parallel_replicas_custom_key/__init__.py similarity index 100% rename from tests/integration/test_async_insert_memory/__init__.py rename to tests/integration/test_parallel_replicas_custom_key/__init__.py diff --git a/tests/integration/test_parallel_replicas_custom_key/configs/remote_servers.xml b/tests/integration/test_parallel_replicas_custom_key/configs/remote_servers.xml new file mode 100644 index 00000000000..308db461498 --- /dev/null +++ b/tests/integration/test_parallel_replicas_custom_key/configs/remote_servers.xml @@ -0,0 +1,50 @@ + + + + + false + + n1 + 9000 + + + n2 + 9000 + + + + false + + n3 + 9000 + + + n4 + 9000 + + + + + + false + + n1 + 9000 + + + n2 + 9000 + + + n3 + 9000 + + + n4 + 9000 + + + + + + diff --git a/tests/integration/test_parallel_replicas_custom_key/test.py b/tests/integration/test_parallel_replicas_custom_key/test.py new file mode 100644 index 00000000000..baac2661506 --- /dev/null +++ b/tests/integration/test_parallel_replicas_custom_key/test.py @@ -0,0 +1,94 @@ +import pytest +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) + +nodes = [ + cluster.add_instance( + f"n{i}", main_configs=["configs/remote_servers.xml"], with_zookeeper=True + ) + for i in range(1, 5) +] + + +@pytest.fixture(scope="module", autouse=True) +def start_cluster(): + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + + +def create_tables(cluster): + n1 = nodes[0] + n1.query("DROP TABLE IF EXISTS dist_table") + n1.query(f"DROP TABLE IF EXISTS test_table ON CLUSTER {cluster}") + + n1.query( + f"CREATE TABLE test_table ON CLUSTER {cluster} (key Int32, value String) Engine=MergeTree ORDER BY (key, sipHash64(value))" + ) + n1.query( + f""" + CREATE TABLE dist_table AS test_table + Engine=Distributed( + {cluster}, + currentDatabase(), + test_table, + rand() + ) + """ + ) + + +def insert_data(cluster, row_num): + create_tables(cluster) + n1 = nodes[0] + n1.query( + f"INSERT INTO dist_table SELECT number % 4, number FROM numbers({row_num})" + ) + n1.query("SYSTEM FLUSH DISTRIBUTED dist_table") + + +@pytest.mark.parametrize("custom_key", ["sipHash64(key)", "key"]) +@pytest.mark.parametrize("filter_type", ["default", "range"]) +@pytest.mark.parametrize( + "cluster", + ["test_multiple_shards_multiple_replicas", "test_single_shard_multiple_replicas"], +) +def test_parallel_replicas_custom_key(start_cluster, cluster, custom_key, filter_type): + for node in nodes: + node.rotate_logs() + + row_num = 1000 + insert_data(cluster, row_num) + + expected_result = "" + for i in range(4): + expected_result += f"{i}\t250\n" + + n1 = nodes[0] + assert ( + n1.query( + "SELECT key, count() FROM dist_table GROUP BY key ORDER BY key", + settings={ + "prefer_localhost_replica": 0, + "max_parallel_replicas": 4, + "parallel_replicas_custom_key": custom_key, + "parallel_replicas_custom_key_filter_type": filter_type, + }, + ) + == expected_result + ) + + if cluster == "test_multiple_shards_multiple_replicas": + # we simply process query on all replicas for each shard by appending the filter on replica + assert all( + node.contains_in_log("Processing query on a replica using custom_key") + for node in nodes + ) + else: + # we first transform all replicas into shards and then append for each shard filter + assert n1.contains_in_log( + "Single shard cluster used with custom_key, transforming replicas into virtual shards" + ) diff --git a/tests/integration/test_replicated_database/test.py b/tests/integration/test_replicated_database/test.py index ead9a762b1b..2ab2fe499ff 100644 --- a/tests/integration/test_replicated_database/test.py +++ b/tests/integration/test_replicated_database/test.py @@ -80,15 +80,15 @@ def started_cluster(): def test_create_replicated_table(started_cluster): main_node.query( - "CREATE DATABASE testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard1', 'replica' || '1');" + "CREATE DATABASE create_replicated_table ENGINE = Replicated('/test/create_replicated_table', 'shard1', 'replica' || '1');" ) dummy_node.query( - "CREATE DATABASE testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard1', 'replica2');" + "CREATE DATABASE create_replicated_table ENGINE = Replicated('/test/create_replicated_table', 'shard1', 'replica2');" ) assert ( "Explicit zookeeper_path and replica_name are specified" in main_node.query_and_get_error( - "CREATE TABLE testdb.replicated_table (d Date, k UInt64, i32 Int32) " + "CREATE TABLE create_replicated_table.replicated_table (d Date, k UInt64, i32 Int32) " "ENGINE=ReplicatedMergeTree('/test/tmp', 'r') ORDER BY k PARTITION BY toYYYYMM(d);" ) ) @@ -96,7 +96,7 @@ def test_create_replicated_table(started_cluster): assert ( "Explicit zookeeper_path and replica_name are specified" in main_node.query_and_get_error( - "CREATE TABLE testdb.replicated_table (d Date, k UInt64, i32 Int32) " + "CREATE TABLE create_replicated_table.replicated_table (d Date, k UInt64, i32 Int32) " "ENGINE=ReplicatedMergeTree('/test/tmp', 'r') ORDER BY k PARTITION BY toYYYYMM(d);" ) ) @@ -104,39 +104,41 @@ def test_create_replicated_table(started_cluster): assert ( "This syntax for *MergeTree engine is deprecated" in main_node.query_and_get_error( - "CREATE TABLE testdb.replicated_table (d Date, k UInt64, i32 Int32) " + "CREATE TABLE create_replicated_table.replicated_table (d Date, k UInt64, i32 Int32) " "ENGINE=ReplicatedMergeTree('/test/tmp/{shard}', '{replica}', d, k, 8192);" ) ) main_node.query( - "CREATE TABLE testdb.replicated_table (d Date, k UInt64, i32 Int32) ENGINE=ReplicatedMergeTree ORDER BY k PARTITION BY toYYYYMM(d);" + "CREATE TABLE create_replicated_table.replicated_table (d Date, k UInt64, i32 Int32) ENGINE=ReplicatedMergeTree ORDER BY k PARTITION BY toYYYYMM(d);" ) expected = ( - "CREATE TABLE testdb.replicated_table\\n(\\n `d` Date,\\n `k` UInt64,\\n `i32` Int32\\n)\\n" + "CREATE TABLE create_replicated_table.replicated_table\\n(\\n `d` Date,\\n `k` UInt64,\\n `i32` Int32\\n)\\n" "ENGINE = ReplicatedMergeTree(\\'/clickhouse/tables/{uuid}/{shard}\\', \\'{replica}\\')\\n" "PARTITION BY toYYYYMM(d)\\nORDER BY k\\nSETTINGS index_granularity = 8192" ) - assert_create_query([main_node, dummy_node], "testdb.replicated_table", expected) - # assert without replacing uuid - assert main_node.query("show create testdb.replicated_table") == dummy_node.query( - "show create testdb.replicated_table" + assert_create_query( + [main_node, dummy_node], "create_replicated_table.replicated_table", expected ) - main_node.query("DROP DATABASE testdb SYNC") - dummy_node.query("DROP DATABASE testdb SYNC") + # assert without replacing uuid + assert main_node.query( + "show create create_replicated_table.replicated_table" + ) == dummy_node.query("show create create_replicated_table.replicated_table") + main_node.query("DROP DATABASE create_replicated_table SYNC") + dummy_node.query("DROP DATABASE create_replicated_table SYNC") @pytest.mark.parametrize("engine", ["MergeTree", "ReplicatedMergeTree"]) def test_simple_alter_table(started_cluster, engine): main_node.query( - "CREATE DATABASE testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard1', 'replica1');" + "CREATE DATABASE test_simple_alter_table ENGINE = Replicated('/test/simple_alter_table', 'shard1', 'replica1');" ) dummy_node.query( - "CREATE DATABASE testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard1', 'replica2');" + "CREATE DATABASE test_simple_alter_table ENGINE = Replicated('/test/simple_alter_table', 'shard1', 'replica2');" ) # test_simple_alter_table - name = "testdb.alter_test_{}".format(engine) + name = "test_simple_alter_table.alter_test_{}".format(engine) main_node.query( "CREATE TABLE {} " "(CounterID UInt32, StartDate Date, UserID UInt32, VisitID UInt32, NestedColumn Nested(A UInt8, S String), ToDrop UInt32) " @@ -184,10 +186,10 @@ def test_simple_alter_table(started_cluster, engine): # test_create_replica_after_delay competing_node.query( - "CREATE DATABASE IF NOT EXISTS testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard1', 'replica3');" + "CREATE DATABASE IF NOT EXISTS test_simple_alter_table ENGINE = Replicated('/test/simple_alter_table', 'shard1', 'replica3');" ) - name = "testdb.alter_test_{}".format(engine) + name = "test_simple_alter_table.alter_test_{}".format(engine) main_node.query("ALTER TABLE {} ADD COLUMN Added3 UInt32;".format(name)) main_node.query("ALTER TABLE {} DROP COLUMN AddedNested1;".format(name)) main_node.query("ALTER TABLE {} RENAME COLUMN Added1 TO AddedNested1;".format(name)) @@ -207,21 +209,21 @@ def test_simple_alter_table(started_cluster, engine): ) assert_create_query([main_node, dummy_node, competing_node], name, expected) - main_node.query("DROP DATABASE testdb SYNC") - dummy_node.query("DROP DATABASE testdb SYNC") - competing_node.query("DROP DATABASE testdb SYNC") + main_node.query("DROP DATABASE test_simple_alter_table SYNC") + dummy_node.query("DROP DATABASE test_simple_alter_table SYNC") + competing_node.query("DROP DATABASE test_simple_alter_table SYNC") @pytest.mark.parametrize("engine", ["MergeTree", "ReplicatedMergeTree"]) def test_delete_from_table(started_cluster, engine): main_node.query( - "CREATE DATABASE testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard1', 'replica1');" + "CREATE DATABASE delete_from_table ENGINE = Replicated('/test/simple_alter_table', 'shard1', 'replica1');" ) dummy_node.query( - "CREATE DATABASE testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard2', 'replica1');" + "CREATE DATABASE delete_from_table ENGINE = Replicated('/test/simple_alter_table', 'shard2', 'replica1');" ) - name = "testdb.delete_test_{}".format(engine) + name = "delete_from_table.delete_test_{}".format(engine) main_node.query( "CREATE TABLE {} " "(id UInt64, value String) " @@ -238,7 +240,7 @@ def test_delete_from_table(started_cluster, engine): table_for_select = name if not "Replicated" in engine: - table_for_select = "cluster('testdb', {})".format(name) + table_for_select = "cluster('delete_from_table', {})".format(name) for node in [main_node, dummy_node]: assert_eq_with_retry( node, @@ -246,8 +248,8 @@ def test_delete_from_table(started_cluster, engine): expected, ) - main_node.query("DROP DATABASE testdb SYNC") - dummy_node.query("DROP DATABASE testdb SYNC") + main_node.query("DROP DATABASE delete_from_table SYNC") + dummy_node.query("DROP DATABASE delete_from_table SYNC") def get_table_uuid(database, name): @@ -276,17 +278,17 @@ def fixture_attachable_part(started_cluster): @pytest.mark.parametrize("engine", ["MergeTree", "ReplicatedMergeTree"]) def test_alter_attach(started_cluster, attachable_part, engine): main_node.query( - "CREATE DATABASE testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard1', 'replica1');" + "CREATE DATABASE alter_attach ENGINE = Replicated('/test/alter_attach', 'shard1', 'replica1');" ) dummy_node.query( - "CREATE DATABASE testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard1', 'replica2');" + "CREATE DATABASE alter_attach ENGINE = Replicated('/test/alter_attach', 'shard1', 'replica2');" ) name = "alter_attach_test_{}".format(engine) main_node.query( - f"CREATE TABLE testdb.{name} (CounterID UInt32) ENGINE = {engine} ORDER BY (CounterID)" + f"CREATE TABLE alter_attach.{name} (CounterID UInt32) ENGINE = {engine} ORDER BY (CounterID)" ) - table_uuid = get_table_uuid("testdb", name) + table_uuid = get_table_uuid("alter_attach", name) # Provide and attach a part to the main node shutil.copytree( attachable_part, @@ -295,113 +297,122 @@ def test_alter_attach(started_cluster, attachable_part, engine): f"database/store/{table_uuid[:3]}/{table_uuid}/detached/all_1_1_0", ), ) - main_node.query(f"ALTER TABLE testdb.{name} ATTACH PART 'all_1_1_0'") + main_node.query(f"ALTER TABLE alter_attach.{name} ATTACH PART 'all_1_1_0'") # On the main node, data is attached - assert main_node.query(f"SELECT CounterID FROM testdb.{name}") == "123\n" + assert main_node.query(f"SELECT CounterID FROM alter_attach.{name}") == "123\n" # On the other node, data is replicated only if using a Replicated table engine if engine == "ReplicatedMergeTree": - assert dummy_node.query(f"SELECT CounterID FROM testdb.{name}") == "123\n" + assert dummy_node.query(f"SELECT CounterID FROM alter_attach.{name}") == "123\n" else: - assert dummy_node.query(f"SELECT CounterID FROM testdb.{name}") == "" - main_node.query("DROP DATABASE testdb SYNC") - dummy_node.query("DROP DATABASE testdb SYNC") + assert dummy_node.query(f"SELECT CounterID FROM alter_attach.{name}") == "" + main_node.query("DROP DATABASE alter_attach SYNC") + dummy_node.query("DROP DATABASE alter_attach SYNC") @pytest.mark.parametrize("engine", ["MergeTree", "ReplicatedMergeTree"]) def test_alter_drop_part(started_cluster, engine): main_node.query( - "CREATE DATABASE testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard1', 'replica1');" + "CREATE DATABASE alter_drop_part ENGINE = Replicated('/test/alter_drop_part', 'shard1', 'replica1');" ) dummy_node.query( - "CREATE DATABASE testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard1', 'replica2');" + "CREATE DATABASE alter_drop_part ENGINE = Replicated('/test/alter_drop_part', 'shard1', 'replica2');" ) table = f"alter_drop_{engine}" part_name = "all_0_0_0" if engine == "ReplicatedMergeTree" else "all_1_1_0" main_node.query( - f"CREATE TABLE testdb.{table} (CounterID UInt32) ENGINE = {engine} ORDER BY (CounterID)" + f"CREATE TABLE alter_drop_part.{table} (CounterID UInt32) ENGINE = {engine} ORDER BY (CounterID)" ) - main_node.query(f"INSERT INTO testdb.{table} VALUES (123)") + main_node.query(f"INSERT INTO alter_drop_part.{table} VALUES (123)") if engine == "MergeTree": - dummy_node.query(f"INSERT INTO testdb.{table} VALUES (456)") - main_node.query(f"ALTER TABLE testdb.{table} DROP PART '{part_name}'") - assert main_node.query(f"SELECT CounterID FROM testdb.{table}") == "" + dummy_node.query(f"INSERT INTO alter_drop_part.{table} VALUES (456)") + main_node.query(f"ALTER TABLE alter_drop_part.{table} DROP PART '{part_name}'") + assert main_node.query(f"SELECT CounterID FROM alter_drop_part.{table}") == "" if engine == "ReplicatedMergeTree": # The DROP operation is still replicated at the table engine level - assert dummy_node.query(f"SELECT CounterID FROM testdb.{table}") == "" + assert dummy_node.query(f"SELECT CounterID FROM alter_drop_part.{table}") == "" else: - assert dummy_node.query(f"SELECT CounterID FROM testdb.{table}") == "456\n" - main_node.query("DROP DATABASE testdb SYNC") - dummy_node.query("DROP DATABASE testdb SYNC") + assert ( + dummy_node.query(f"SELECT CounterID FROM alter_drop_part.{table}") + == "456\n" + ) + main_node.query("DROP DATABASE alter_drop_part SYNC") + dummy_node.query("DROP DATABASE alter_drop_part SYNC") @pytest.mark.parametrize("engine", ["MergeTree", "ReplicatedMergeTree"]) def test_alter_detach_part(started_cluster, engine): main_node.query( - "CREATE DATABASE testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard1', 'replica1');" + "CREATE DATABASE alter_detach_part ENGINE = Replicated('/test/alter_detach_part', 'shard1', 'replica1');" ) dummy_node.query( - "CREATE DATABASE testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard1', 'replica2');" + "CREATE DATABASE alter_detach_part ENGINE = Replicated('/test/alter_detach_part', 'shard1', 'replica2');" ) table = f"alter_detach_{engine}" part_name = "all_0_0_0" if engine == "ReplicatedMergeTree" else "all_1_1_0" main_node.query( - f"CREATE TABLE testdb.{table} (CounterID UInt32) ENGINE = {engine} ORDER BY (CounterID)" + f"CREATE TABLE alter_detach_part.{table} (CounterID UInt32) ENGINE = {engine} ORDER BY (CounterID)" ) - main_node.query(f"INSERT INTO testdb.{table} VALUES (123)") + main_node.query(f"INSERT INTO alter_detach_part.{table} VALUES (123)") if engine == "MergeTree": - dummy_node.query(f"INSERT INTO testdb.{table} VALUES (456)") - main_node.query(f"ALTER TABLE testdb.{table} DETACH PART '{part_name}'") - detached_parts_query = f"SELECT name FROM system.detached_parts WHERE database='testdb' AND table='{table}'" + dummy_node.query(f"INSERT INTO alter_detach_part.{table} VALUES (456)") + main_node.query(f"ALTER TABLE alter_detach_part.{table} DETACH PART '{part_name}'") + detached_parts_query = f"SELECT name FROM system.detached_parts WHERE database='alter_detach_part' AND table='{table}'" assert main_node.query(detached_parts_query) == f"{part_name}\n" if engine == "ReplicatedMergeTree": # The detach operation is still replicated at the table engine level assert dummy_node.query(detached_parts_query) == f"{part_name}\n" else: assert dummy_node.query(detached_parts_query) == "" - main_node.query("DROP DATABASE testdb SYNC") - dummy_node.query("DROP DATABASE testdb SYNC") + main_node.query("DROP DATABASE alter_detach_part SYNC") + dummy_node.query("DROP DATABASE alter_detach_part SYNC") @pytest.mark.parametrize("engine", ["MergeTree", "ReplicatedMergeTree"]) def test_alter_drop_detached_part(started_cluster, engine): main_node.query( - "CREATE DATABASE testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard1', 'replica1');" + "CREATE DATABASE alter_drop_detached_part ENGINE = Replicated('/test/alter_drop_detached_part', 'shard1', 'replica1');" ) dummy_node.query( - "CREATE DATABASE testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard1', 'replica2');" + "CREATE DATABASE alter_drop_detached_part ENGINE = Replicated('/test/alter_drop_detached_part', 'shard1', 'replica2');" ) table = f"alter_drop_detached_{engine}" part_name = "all_0_0_0" if engine == "ReplicatedMergeTree" else "all_1_1_0" main_node.query( - f"CREATE TABLE testdb.{table} (CounterID UInt32) ENGINE = {engine} ORDER BY (CounterID)" + f"CREATE TABLE alter_drop_detached_part.{table} (CounterID UInt32) ENGINE = {engine} ORDER BY (CounterID)" + ) + main_node.query(f"INSERT INTO alter_drop_detached_part.{table} VALUES (123)") + main_node.query( + f"ALTER TABLE alter_drop_detached_part.{table} DETACH PART '{part_name}'" ) - main_node.query(f"INSERT INTO testdb.{table} VALUES (123)") - main_node.query(f"ALTER TABLE testdb.{table} DETACH PART '{part_name}'") if engine == "MergeTree": - dummy_node.query(f"INSERT INTO testdb.{table} VALUES (456)") - dummy_node.query(f"ALTER TABLE testdb.{table} DETACH PART '{part_name}'") - main_node.query(f"ALTER TABLE testdb.{table} DROP DETACHED PART '{part_name}'") - detached_parts_query = f"SELECT name FROM system.detached_parts WHERE database='testdb' AND table='{table}'" + dummy_node.query(f"INSERT INTO alter_drop_detached_part.{table} VALUES (456)") + dummy_node.query( + f"ALTER TABLE alter_drop_detached_part.{table} DETACH PART '{part_name}'" + ) + main_node.query( + f"ALTER TABLE alter_drop_detached_part.{table} DROP DETACHED PART '{part_name}'" + ) + detached_parts_query = f"SELECT name FROM system.detached_parts WHERE database='alter_drop_detached_part' AND table='{table}'" assert main_node.query(detached_parts_query) == "" assert dummy_node.query(detached_parts_query) == f"{part_name}\n" - main_node.query("DROP DATABASE testdb SYNC") - dummy_node.query("DROP DATABASE testdb SYNC") + main_node.query("DROP DATABASE alter_drop_detached_part SYNC") + dummy_node.query("DROP DATABASE alter_drop_detached_part SYNC") @pytest.mark.parametrize("engine", ["MergeTree", "ReplicatedMergeTree"]) def test_alter_drop_partition(started_cluster, engine): main_node.query( - "CREATE DATABASE alter_drop_partition ENGINE = Replicated('/clickhouse/databases/test_alter_drop_partition', 'shard1', 'replica1');" + "CREATE DATABASE alter_drop_partition ENGINE = Replicated('/test/alter_drop_partition', 'shard1', 'replica1');" ) dummy_node.query( - "CREATE DATABASE alter_drop_partition ENGINE = Replicated('/clickhouse/databases/test_alter_drop_partition', 'shard1', 'replica2');" + "CREATE DATABASE alter_drop_partition ENGINE = Replicated('/test/alter_drop_partition', 'shard1', 'replica2');" ) snapshotting_node.query( - "CREATE DATABASE alter_drop_partition ENGINE = Replicated('/clickhouse/databases/test_alter_drop_partition', 'shard2', 'replica1');" + "CREATE DATABASE alter_drop_partition ENGINE = Replicated('/test/alter_drop_partition', 'shard2', 'replica1');" ) table = f"alter_drop_partition.alter_drop_{engine}" @@ -430,52 +441,52 @@ def test_alter_drop_partition(started_cluster, engine): def test_alter_fetch(started_cluster): main_node.query( - "CREATE DATABASE testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard1', 'replica1');" + "CREATE DATABASE alter_fetch ENGINE = Replicated('/test/alter_fetch', 'shard1', 'replica1');" ) dummy_node.query( - "CREATE DATABASE testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard1', 'replica2');" + "CREATE DATABASE alter_fetch ENGINE = Replicated('/test/alter_fetch', 'shard1', 'replica2');" ) main_node.query( - "CREATE TABLE testdb.fetch_source (CounterID UInt32) ENGINE = ReplicatedMergeTree ORDER BY (CounterID)" + "CREATE TABLE alter_fetch.fetch_source (CounterID UInt32) ENGINE = ReplicatedMergeTree ORDER BY (CounterID)" ) main_node.query( - "CREATE TABLE testdb.fetch_target (CounterID UInt32) ENGINE = ReplicatedMergeTree ORDER BY (CounterID)" + "CREATE TABLE alter_fetch.fetch_target (CounterID UInt32) ENGINE = ReplicatedMergeTree ORDER BY (CounterID)" ) - main_node.query("INSERT INTO testdb.fetch_source VALUES (123)") - table_uuid = get_table_uuid("testdb", "fetch_source") + main_node.query("INSERT INTO alter_fetch.fetch_source VALUES (123)") + table_uuid = get_table_uuid("alter_fetch", "fetch_source") main_node.query( - f"ALTER TABLE testdb.fetch_target FETCH PART 'all_0_0_0' FROM '/clickhouse/tables/{table_uuid}/{{shard}}' " + f"ALTER TABLE alter_fetch.fetch_target FETCH PART 'all_0_0_0' FROM '/clickhouse/tables/{table_uuid}/{{shard}}' " ) - detached_parts_query = "SELECT name FROM system.detached_parts WHERE database='testdb' AND table='fetch_target'" + detached_parts_query = "SELECT name FROM system.detached_parts WHERE database='alter_fetch' AND table='fetch_target'" assert main_node.query(detached_parts_query) == "all_0_0_0\n" assert dummy_node.query(detached_parts_query) == "" - main_node.query("DROP DATABASE testdb SYNC") - dummy_node.query("DROP DATABASE testdb SYNC") + main_node.query("DROP DATABASE alter_fetch SYNC") + dummy_node.query("DROP DATABASE alter_fetch SYNC") def test_alters_from_different_replicas(started_cluster): main_node.query( - "CREATE DATABASE testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard1', 'replica1');" + "CREATE DATABASE alters_from_different_replicas ENGINE = Replicated('/test/alters_from_different_replicas', 'shard1', 'replica1');" ) dummy_node.query( - "CREATE DATABASE testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard1', 'replica2');" + "CREATE DATABASE alters_from_different_replicas ENGINE = Replicated('/test/alters_from_different_replicas', 'shard1', 'replica2');" ) # test_alters_from_different_replicas competing_node.query( - "CREATE DATABASE testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard1', 'replica3');" + "CREATE DATABASE alters_from_different_replicas ENGINE = Replicated('/test/alters_from_different_replicas', 'shard1', 'replica3');" ) main_node.query( - "CREATE TABLE testdb.concurrent_test " + "CREATE TABLE alters_from_different_replicas.concurrent_test " "(CounterID UInt32, StartDate Date, UserID UInt32, VisitID UInt32, NestedColumn Nested(A UInt8, S String), ToDrop UInt32) " "ENGINE = MergeTree PARTITION BY toYYYYMM(StartDate) ORDER BY (CounterID, StartDate, intHash32(UserID), VisitID);" ) main_node.query( - "CREATE TABLE testdb.dist AS testdb.concurrent_test ENGINE = Distributed(testdb, testdb, concurrent_test, CounterID)" + "CREATE TABLE alters_from_different_replicas.dist AS alters_from_different_replicas.concurrent_test ENGINE = Distributed(alters_from_different_replicas, alters_from_different_replicas, concurrent_test, CounterID)" ) dummy_node.stop_clickhouse(kill=True) @@ -484,7 +495,7 @@ def test_alters_from_different_replicas(started_cluster): assert ( "There are 1 unfinished hosts (0 of them are currently active)" in competing_node.query_and_get_error( - "ALTER TABLE testdb.concurrent_test ADD COLUMN Added0 UInt32;", + "ALTER TABLE alters_from_different_replicas.concurrent_test ADD COLUMN Added0 UInt32;", settings=settings, ) ) @@ -493,7 +504,7 @@ def test_alters_from_different_replicas(started_cluster): "distributed_ddl_output_mode": "null_status_on_timeout", } assert "shard1\treplica2\tQUEUED\t" in main_node.query( - "ALTER TABLE testdb.concurrent_test ADD COLUMN Added2 UInt32;", + "ALTER TABLE alters_from_different_replicas.concurrent_test ADD COLUMN Added2 UInt32;", settings=settings, ) settings = { @@ -501,22 +512,22 @@ def test_alters_from_different_replicas(started_cluster): "distributed_ddl_output_mode": "never_throw", } assert "shard1\treplica2\tQUEUED\t" in competing_node.query( - "ALTER TABLE testdb.concurrent_test ADD COLUMN Added1 UInt32 AFTER Added0;", + "ALTER TABLE alters_from_different_replicas.concurrent_test ADD COLUMN Added1 UInt32 AFTER Added0;", settings=settings, ) dummy_node.start_clickhouse() main_node.query( - "ALTER TABLE testdb.concurrent_test ADD COLUMN AddedNested1 Nested(A UInt32, B UInt64) AFTER Added2;" + "ALTER TABLE alters_from_different_replicas.concurrent_test ADD COLUMN AddedNested1 Nested(A UInt32, B UInt64) AFTER Added2;" ) competing_node.query( - "ALTER TABLE testdb.concurrent_test ADD COLUMN AddedNested1.C Array(String) AFTER AddedNested1.B;" + "ALTER TABLE alters_from_different_replicas.concurrent_test ADD COLUMN AddedNested1.C Array(String) AFTER AddedNested1.B;" ) main_node.query( - "ALTER TABLE testdb.concurrent_test ADD COLUMN AddedNested2 Nested(A UInt32, B UInt64) AFTER AddedNested1;" + "ALTER TABLE alters_from_different_replicas.concurrent_test ADD COLUMN AddedNested2 Nested(A UInt32, B UInt64) AFTER AddedNested1;" ) expected = ( - "CREATE TABLE testdb.concurrent_test\\n(\\n `CounterID` UInt32,\\n `StartDate` Date,\\n `UserID` UInt32,\\n" + "CREATE TABLE alters_from_different_replicas.concurrent_test\\n(\\n `CounterID` UInt32,\\n `StartDate` Date,\\n `UserID` UInt32,\\n" " `VisitID` UInt32,\\n `NestedColumn.A` Array(UInt8),\\n `NestedColumn.S` Array(String),\\n `ToDrop` UInt32,\\n" " `Added0` UInt32,\\n `Added1` UInt32,\\n `Added2` UInt32,\\n `AddedNested1.A` Array(UInt32),\\n" " `AddedNested1.B` Array(UInt64),\\n `AddedNested1.C` Array(String),\\n `AddedNested2.A` Array(UInt32),\\n" @@ -524,51 +535,63 @@ def test_alters_from_different_replicas(started_cluster): "ENGINE = MergeTree\\nPARTITION BY toYYYYMM(StartDate)\\nORDER BY (CounterID, StartDate, intHash32(UserID), VisitID)\\nSETTINGS index_granularity = 8192" ) - assert_create_query([main_node, competing_node], "testdb.concurrent_test", expected) + assert_create_query( + [main_node, competing_node], + "alters_from_different_replicas.concurrent_test", + expected, + ) # test_create_replica_after_delay - main_node.query("DROP TABLE testdb.concurrent_test SYNC") + main_node.query("DROP TABLE alters_from_different_replicas.concurrent_test SYNC") main_node.query( - "CREATE TABLE testdb.concurrent_test " + "CREATE TABLE alters_from_different_replicas.concurrent_test " "(CounterID UInt32, StartDate Date, UserID UInt32, VisitID UInt32, NestedColumn Nested(A UInt8, S String), ToDrop UInt32) " "ENGINE = ReplicatedMergeTree ORDER BY CounterID;" ) expected = ( - "CREATE TABLE testdb.concurrent_test\\n(\\n `CounterID` UInt32,\\n `StartDate` Date,\\n `UserID` UInt32,\\n" + "CREATE TABLE alters_from_different_replicas.concurrent_test\\n(\\n `CounterID` UInt32,\\n `StartDate` Date,\\n `UserID` UInt32,\\n" " `VisitID` UInt32,\\n `NestedColumn.A` Array(UInt8),\\n `NestedColumn.S` Array(String),\\n `ToDrop` UInt32\\n)\\n" "ENGINE = ReplicatedMergeTree(\\'/clickhouse/tables/{uuid}/{shard}\\', \\'{replica}\\')\\nORDER BY CounterID\\nSETTINGS index_granularity = 8192" ) - assert_create_query([main_node, competing_node], "testdb.concurrent_test", expected) + assert_create_query( + [main_node, competing_node], + "alters_from_different_replicas.concurrent_test", + expected, + ) main_node.query( - "INSERT INTO testdb.dist (CounterID, StartDate, UserID) SELECT number, addDays(toDate('2020-02-02'), number), intHash32(number) FROM numbers(10)" + "INSERT INTO alters_from_different_replicas.dist (CounterID, StartDate, UserID) SELECT number, addDays(toDate('2020-02-02'), number), intHash32(number) FROM numbers(10)" ) # test_replica_restart main_node.restart_clickhouse() expected = ( - "CREATE TABLE testdb.concurrent_test\\n(\\n `CounterID` UInt32,\\n `StartDate` Date,\\n `UserID` UInt32,\\n" + "CREATE TABLE alters_from_different_replicas.concurrent_test\\n(\\n `CounterID` UInt32,\\n `StartDate` Date,\\n `UserID` UInt32,\\n" " `VisitID` UInt32,\\n `NestedColumn.A` Array(UInt8),\\n `NestedColumn.S` Array(String),\\n `ToDrop` UInt32\\n)\\n" "ENGINE = ReplicatedMergeTree(\\'/clickhouse/tables/{uuid}/{shard}\\', \\'{replica}\\')\\nORDER BY CounterID\\nSETTINGS index_granularity = 8192" ) # test_snapshot_and_snapshot_recover snapshotting_node.query( - "CREATE DATABASE testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard2', 'replica1');" + "CREATE DATABASE alters_from_different_replicas ENGINE = Replicated('/test/alters_from_different_replicas', 'shard2', 'replica1');" ) snapshot_recovering_node.query( - "CREATE DATABASE testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard2', 'replica2');" + "CREATE DATABASE alters_from_different_replicas ENGINE = Replicated('/test/alters_from_different_replicas', 'shard2', 'replica2');" + ) + assert_create_query( + all_nodes, "alters_from_different_replicas.concurrent_test", expected ) - assert_create_query(all_nodes, "testdb.concurrent_test", expected) - main_node.query("SYSTEM FLUSH DISTRIBUTED testdb.dist") + main_node.query("SYSTEM FLUSH DISTRIBUTED alters_from_different_replicas.dist") main_node.query( - "ALTER TABLE testdb.concurrent_test UPDATE StartDate = addYears(StartDate, 1) WHERE 1" + "ALTER TABLE alters_from_different_replicas.concurrent_test UPDATE StartDate = addYears(StartDate, 1) WHERE 1" + ) + res = main_node.query( + "ALTER TABLE alters_from_different_replicas.concurrent_test DELETE WHERE UserID % 2" ) - res = main_node.query("ALTER TABLE testdb.concurrent_test DELETE WHERE UserID % 2") assert ( "shard1\treplica1\tOK" in res and "shard1\treplica2\tOK" in res @@ -585,28 +608,34 @@ def test_alters_from_different_replicas(started_cluster): ) assert ( main_node.query( - "SELECT shard_num, replica_num, host_name FROM system.clusters WHERE cluster='testdb'" + "SELECT shard_num, replica_num, host_name FROM system.clusters WHERE cluster='alters_from_different_replicas'" ) == expected ) # test_drop_and_create_replica - main_node.query("DROP DATABASE testdb SYNC") + main_node.query("DROP DATABASE alters_from_different_replicas SYNC") main_node.query( - "CREATE DATABASE testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard1', 'replica1');" + "CREATE DATABASE alters_from_different_replicas ENGINE = Replicated('/test/alters_from_different_replicas', 'shard1', 'replica1');" ) expected = ( - "CREATE TABLE testdb.concurrent_test\\n(\\n `CounterID` UInt32,\\n `StartDate` Date,\\n `UserID` UInt32,\\n" + "CREATE TABLE alters_from_different_replicas.concurrent_test\\n(\\n `CounterID` UInt32,\\n `StartDate` Date,\\n `UserID` UInt32,\\n" " `VisitID` UInt32,\\n `NestedColumn.A` Array(UInt8),\\n `NestedColumn.S` Array(String),\\n `ToDrop` UInt32\\n)\\n" "ENGINE = ReplicatedMergeTree(\\'/clickhouse/tables/{uuid}/{shard}\\', \\'{replica}\\')\\nORDER BY CounterID\\nSETTINGS index_granularity = 8192" ) - assert_create_query([main_node, competing_node], "testdb.concurrent_test", expected) - assert_create_query(all_nodes, "testdb.concurrent_test", expected) + assert_create_query( + [main_node, competing_node], + "alters_from_different_replicas.concurrent_test", + expected, + ) + assert_create_query( + all_nodes, "alters_from_different_replicas.concurrent_test", expected + ) for node in all_nodes: - node.query("SYSTEM SYNC REPLICA testdb.concurrent_test") + node.query("SYSTEM SYNC REPLICA alters_from_different_replicas.concurrent_test") expected = ( "0\t2021-02-02\t4249604106\n" @@ -618,14 +647,14 @@ def test_alters_from_different_replicas(started_cluster): assert_eq_with_retry( dummy_node, - "SELECT CounterID, StartDate, UserID FROM testdb.dist ORDER BY CounterID", + "SELECT CounterID, StartDate, UserID FROM alters_from_different_replicas.dist ORDER BY CounterID", expected, ) - main_node.query("DROP DATABASE testdb SYNC") - dummy_node.query("DROP DATABASE testdb SYNC") - competing_node.query("DROP DATABASE testdb SYNC") - snapshotting_node.query("DROP DATABASE testdb SYNC") - snapshot_recovering_node.query("DROP DATABASE testdb SYNC") + main_node.query("DROP DATABASE alters_from_different_replicas SYNC") + dummy_node.query("DROP DATABASE alters_from_different_replicas SYNC") + competing_node.query("DROP DATABASE alters_from_different_replicas SYNC") + snapshotting_node.query("DROP DATABASE alters_from_different_replicas SYNC") + snapshot_recovering_node.query("DROP DATABASE alters_from_different_replicas SYNC") def create_some_tables(db): @@ -1063,10 +1092,10 @@ def test_server_uuid(started_cluster): def test_sync_replica(started_cluster): main_node.query( - "CREATE DATABASE test_sync_database ENGINE = Replicated('/clickhouse/databases/test1', 'shard1', 'replica1');" + "CREATE DATABASE test_sync_database ENGINE = Replicated('/test/sync_replica', 'shard1', 'replica1');" ) dummy_node.query( - "CREATE DATABASE test_sync_database ENGINE = Replicated('/clickhouse/databases/test1', 'shard1', 'replica2');" + "CREATE DATABASE test_sync_database ENGINE = Replicated('/test/sync_replica', 'shard1', 'replica2');" ) number_of_tables = 1000 @@ -1113,17 +1142,20 @@ def test_sync_replica(started_cluster): ) lp1 = main_node.query( - "select value from system.zookeeper where path='/clickhouse/databases/test1/replicas/shard1|replica1' and name='log_ptr'" + "select value from system.zookeeper where path='/test/sync_replica/replicas/shard1|replica1' and name='log_ptr'" ) lp2 = main_node.query( - "select value from system.zookeeper where path='/clickhouse/databases/test1/replicas/shard1|replica2' and name='log_ptr'" + "select value from system.zookeeper where path='/test/sync_replica/replicas/shard1|replica2' and name='log_ptr'" ) max_lp = main_node.query( - "select value from system.zookeeper where path='/clickhouse/databases/test1/' and name='max_log_ptr'" + "select value from system.zookeeper where path='/test/sync_replica/' and name='max_log_ptr'" ) assert lp1 == max_lp assert lp2 == max_lp + main_node.query("DROP DATABASE test_sync_database SYNC") + dummy_node.query("DROP DATABASE test_sync_database SYNC") + def test_force_synchronous_settings(started_cluster): main_node.query( diff --git a/tests/integration/test_replicated_merge_tree_s3_zero_copy/test.py b/tests/integration/test_replicated_merge_tree_s3_zero_copy/test.py index f0bc12e3125..5bbd5293453 100644 --- a/tests/integration/test_replicated_merge_tree_s3_zero_copy/test.py +++ b/tests/integration/test_replicated_merge_tree_s3_zero_copy/test.py @@ -194,10 +194,12 @@ def test_drop_table(cluster): ) node.query_with_retry( "system sync replica test_drop_table", - settings={"receive_timeout": 10}, - retry_count=5, + settings={"receive_timeout": 5}, + sleep_time=5, + retry_count=10, ) - node2.query("drop table test_drop_table") + node2.query("drop table test_drop_table sync") assert "1000\t499500\n" == node.query( "select count(n), sum(n) from test_drop_table" ) + node.query("drop table test_drop_table sync") diff --git a/tests/integration/test_storage_kafka/test.py b/tests/integration/test_storage_kafka/test.py index 9f617369859..51952ac1eb7 100644 --- a/tests/integration/test_storage_kafka/test.py +++ b/tests/integration/test_storage_kafka/test.py @@ -285,6 +285,56 @@ def avro_confluent_message(schema_registry_client, value): # Tests +def test_kafka_prohibited_column_types(kafka_cluster): + def assert_returned_exception(e): + assert e.value.returncode == 36 + assert ( + "KafkaEngine doesn't support DEFAULT/MATERIALIZED/EPHEMERAL/ALIAS expressions for columns." + in str(e.value) + ) + + # check column with DEFAULT expression + with pytest.raises(QueryRuntimeException) as exception: + instance.query( + """ + CREATE TABLE test.kafka (a Int, b Int DEFAULT 0) + ENGINE = Kafka('{kafka_broker}:19092', '{kafka_topic_new}', '{kafka_group_name_new}', '{kafka_format_json_each_row}', '\\n') + """ + ) + assert_returned_exception(exception) + + # check EPHEMERAL + with pytest.raises(QueryRuntimeException) as exception: + instance.query( + """ + CREATE TABLE test.kafka (a Int, b Int EPHEMERAL) + ENGINE = Kafka('{kafka_broker}:19092', '{kafka_topic_new}', '{kafka_group_name_new}', '{kafka_format_json_each_row}', '\\n') + """ + ) + assert_returned_exception(exception) + + # check ALIAS + with pytest.raises(QueryRuntimeException) as exception: + instance.query( + """ + CREATE TABLE test.kafka (a Int, b String Alias toString(a)) + ENGINE = Kafka('{kafka_broker}:19092', '{kafka_topic_new}', '{kafka_group_name_new}', '{kafka_format_json_each_row}', '\\n') + """ + ) + assert_returned_exception(exception) + + # check MATERIALIZED + # check ALIAS + with pytest.raises(QueryRuntimeException) as exception: + instance.query( + """ + CREATE TABLE test.kafka (a Int, b String MATERIALIZED toString(a)) + ENGINE = Kafka('{kafka_broker}:19092', '{kafka_topic_new}', '{kafka_group_name_new}', '{kafka_format_json_each_row}', '\\n') + """ + ) + assert_returned_exception(exception) + + def test_kafka_settings_old_syntax(kafka_cluster): assert TSV( instance.query( diff --git a/tests/integration/test_storage_kerberized_hdfs/secrets/krb.conf b/tests/integration/test_storage_kerberized_hdfs/secrets/krb.conf index b43a54b4dc5..dffdcaebe81 100644 --- a/tests/integration/test_storage_kerberized_hdfs/secrets/krb.conf +++ b/tests/integration/test_storage_kerberized_hdfs/secrets/krb.conf @@ -9,6 +9,7 @@ dns_lookup_kdc = false ticket_lifetime = 5s forwardable = true + rdns = false default_tgs_enctypes = des3-hmac-sha1 default_tkt_enctypes = des3-hmac-sha1 permitted_enctypes = des3-hmac-sha1 diff --git a/tests/integration/test_storage_kerberized_kafka/secrets/krb.conf b/tests/integration/test_storage_kerberized_kafka/secrets/krb.conf index 1efdf510f22..bda73a285cf 100644 --- a/tests/integration/test_storage_kerberized_kafka/secrets/krb.conf +++ b/tests/integration/test_storage_kerberized_kafka/secrets/krb.conf @@ -10,6 +10,7 @@ ticket_lifetime = 15s renew_lifetime = 15s forwardable = true + rdns = false [realms] TEST.CLICKHOUSE.TECH = { diff --git a/tests/integration/test_storage_s3/test.py b/tests/integration/test_storage_s3/test.py index 54944b56919..8b20727a7b5 100644 --- a/tests/integration/test_storage_s3/test.py +++ b/tests/integration/test_storage_s3/test.py @@ -1056,13 +1056,13 @@ def test_seekable_formats(started_cluster): table_function = f"s3(s3_orc, structure='a Int32, b String', format='ORC')" exec_query_with_retry( instance, - f"insert into table function {table_function} SELECT number, randomString(100) FROM numbers(1000000) settings s3_truncate_on_insert=1", + f"insert into table function {table_function} SELECT number, randomString(100) FROM numbers(1500000) settings s3_truncate_on_insert=1", ) result = instance.query( - f"SELECT count() FROM {table_function} SETTINGS max_memory_usage='50M'" + f"SELECT count() FROM {table_function} SETTINGS max_memory_usage='60M'" ) - assert int(result) == 1000000 + assert int(result) == 1500000 instance.query(f"SELECT * FROM {table_function} FORMAT Null") @@ -1073,7 +1073,7 @@ def test_seekable_formats(started_cluster): result = result.strip() assert result.endswith("MiB") result = result[: result.index(".")] - assert int(result) > 80 + assert int(result) > 150 def test_seekable_formats_url(started_cluster): @@ -1083,23 +1083,23 @@ def test_seekable_formats_url(started_cluster): table_function = f"s3(s3_parquet, structure='a Int32, b String', format='Parquet')" exec_query_with_retry( instance, - f"insert into table function {table_function} SELECT number, randomString(100) FROM numbers(1000000) settings s3_truncate_on_insert=1", + f"insert into table function {table_function} SELECT number, randomString(100) FROM numbers(1500000) settings s3_truncate_on_insert=1", ) result = instance.query(f"SELECT count() FROM {table_function}") - assert int(result) == 1000000 + assert int(result) == 1500000 table_function = f"s3(s3_orc, structure='a Int32, b String', format='ORC')" exec_query_with_retry( instance, - f"insert into table function {table_function} SELECT number, randomString(100) FROM numbers(1000000) settings s3_truncate_on_insert=1", + f"insert into table function {table_function} SELECT number, randomString(100) FROM numbers(1500000) settings s3_truncate_on_insert=1", ) table_function = f"url('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test_parquet', 'Parquet', 'a Int32, b String')" result = instance.query( - f"SELECT count() FROM {table_function} SETTINGS max_memory_usage='50M'" + f"SELECT count() FROM {table_function} SETTINGS max_memory_usage='60M'" ) - assert int(result) == 1000000 + assert int(result) == 1500000 def test_empty_file(started_cluster): diff --git a/tests/queries/0_stateless/02521_cannot-find-column-in-projection.reference b/tests/integration/test_storage_url_http_headers/__init__.py similarity index 100% rename from tests/queries/0_stateless/02521_cannot-find-column-in-projection.reference rename to tests/integration/test_storage_url_http_headers/__init__.py diff --git a/tests/integration/test_storage_url_http_headers/http_headers_echo_server.py b/tests/integration/test_storage_url_http_headers/http_headers_echo_server.py new file mode 100644 index 00000000000..b1a3f6777b1 --- /dev/null +++ b/tests/integration/test_storage_url_http_headers/http_headers_echo_server.py @@ -0,0 +1,31 @@ +import http.server + +RESULT_PATH = "/headers.txt" + + +class RequestHandler(http.server.BaseHTTPRequestHandler): + def log_message(self, *args): + with open(RESULT_PATH, "w") as f: + f.write(self.headers.as_string()) + + def do_POST(self): + self.rfile.read1() + self.send_response(200) + self.end_headers() + self.wfile.write(b'{"status":"ok"}') + + +if __name__ == "__main__": + with open(RESULT_PATH, "w") as f: + f.write("") + httpd = http.server.HTTPServer( + ( + "localhost", + 8000, + ), + RequestHandler, + ) + try: + httpd.serve_forever() + finally: + httpd.server_close() diff --git a/tests/integration/test_storage_url_http_headers/test.py b/tests/integration/test_storage_url_http_headers/test.py new file mode 100644 index 00000000000..3bbf5ec81c9 --- /dev/null +++ b/tests/integration/test_storage_url_http_headers/test.py @@ -0,0 +1,66 @@ +import pytest +import os +import time + +from . import http_headers_echo_server + +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) +server = cluster.add_instance("node") + + +def run_echo_server(): + script_dir = os.path.dirname(os.path.realpath(__file__)) + + server.copy_file_to_container( + os.path.join(script_dir, "http_headers_echo_server.py"), + "/http_headers_echo_server.py", + ) + + server.exec_in_container( + [ + "bash", + "-c", + "python3 /http_headers_echo_server.py > /http_headers_echo.server.log 2>&1", + ], + detach=True, + user="root", + ) + + for _ in range(0, 10): + ping_response = server.exec_in_container( + ["curl", "-s", f"http://localhost:8000/"], + nothrow=True, + ) + + if "html" in ping_response: + return + + print(ping_response) + + raise Exception("Echo server is not responding") + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + run_echo_server() + yield cluster + finally: + cluster.shutdown() + + +def test_storage_url_http_headers(started_cluster): + query = "INSERT INTO TABLE FUNCTION url('http://localhost:8000/', JSON, 'a UInt64', headers('X-My-Custom-Header'='test-header')) VALUES (1)" + + server.query(query) + + result = server.exec_in_container( + ["cat", http_headers_echo_server.RESULT_PATH], user="root" + ) + + print(result) + + assert "X-My-Custom-Header: test-header" in result diff --git a/tests/integration/test_zero_copy_fetch/test.py b/tests/integration/test_zero_copy_fetch/test.py index f13eac5e9d1..ed6d6117b31 100644 --- a/tests/integration/test_zero_copy_fetch/test.py +++ b/tests/integration/test_zero_copy_fetch/test.py @@ -5,6 +5,7 @@ import random import string import time +from multiprocessing.dummy import Pool import pytest from helpers.cluster import ClickHouseCluster @@ -102,3 +103,79 @@ SETTINGS index_granularity = 8192, storage_policy = 's3'""" assert part_to_disk["20230102_0_0_0"] == "s3" assert part_to_disk["20230109_0_0_0"] == "s3" assert part_to_disk["20230116_0_0_0"] == "default" + + +def test_concurrent_move_to_s3(started_cluster): + node1 = cluster.instances["node1"] + node2 = cluster.instances["node2"] + + node1.query( + """ +CREATE TABLE test_concurrent_move (EventDate Date, CounterID UInt32) +ENGINE = ReplicatedMergeTree('/clickhouse-tables/test_concurrent_move', 'r1') +PARTITION BY CounterID +ORDER BY (CounterID, EventDate) +SETTINGS index_granularity = 8192, storage_policy = 's3'""" + ) + + node2.query( + """ +CREATE TABLE test_concurrent_move (EventDate Date, CounterID UInt32) +ENGINE = ReplicatedMergeTree('/clickhouse-tables/test_concurrent_move', 'r2') +PARTITION BY CounterID +ORDER BY (CounterID, EventDate) +SETTINGS index_granularity = 8192, storage_policy = 's3'""" + ) + partitions = range(10) + + for i in partitions: + node1.query( + f"INSERT INTO test_concurrent_move SELECT toDate('2023-01-01') + toIntervalDay(number), {i} from system.numbers limit 20" + ) + node1.query( + f"INSERT INTO test_concurrent_move SELECT toDate('2023-01-01') + toIntervalDay(number) + rand(), {i} from system.numbers limit 20" + ) + node1.query( + f"INSERT INTO test_concurrent_move SELECT toDate('2023-01-01') + toIntervalDay(number) + rand(), {i} from system.numbers limit 20" + ) + node1.query( + f"INSERT INTO test_concurrent_move SELECT toDate('2023-01-01') + toIntervalDay(number) + rand(), {i} from system.numbers limit 20" + ) + + node2.query("SYSTEM SYNC REPLICA test_concurrent_move") + + # check that we can move parts concurrently without exceptions + p = Pool(3) + for i in partitions: + + def move_partition_to_s3(node): + node.query( + f"ALTER TABLE test_concurrent_move MOVE PARTITION '{i}' TO DISK 's3'" + ) + + j1 = p.apply_async(move_partition_to_s3, (node1,)) + j2 = p.apply_async(move_partition_to_s3, (node2,)) + j1.get() + j2.get() + + def get_part_to_disk(query_result): + part_to_disk = {} + for row in query_result.strip().split("\n"): + disk, part = row.split("\t") + part_to_disk[part] = disk + return part_to_disk + + part_to_disk = get_part_to_disk( + node1.query( + "SELECT disk_name, name FROM system.parts where table = 'test_concurrent_move' and active" + ) + ) + + assert all([value == "s3" for value in part_to_disk.values()]) + + part_to_disk = get_part_to_disk( + node2.query( + "SELECT disk_name, name FROM system.parts where table = 'test_concurrent_move' and active" + ) + ) + assert all([value == "s3" for value in part_to_disk.values()]) diff --git a/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql b/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql index bb2a8ce8b98..12870361cfe 100644 --- a/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql +++ b/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql @@ -58,6 +58,7 @@ select 'number of noisy messages', max2(count(), 10) from (select count() / (sel select 'incorrect patterns', max2(countDistinct(message_format_string), 15) from ( select message_format_string, any(message) as any_message from logs where message not like (replaceRegexpAll(message_format_string, '{[:.0-9dfx]*}', '%') as s) + and message not like (s || ' (skipped % similar messages)') and message not like ('%Exception: '||s||'%') group by message_format_string ) where any_message not like '%Poco::Exception%'; diff --git a/tests/queries/0_stateless/00002_system_numbers.sql b/tests/queries/0_stateless/00002_system_numbers.sql index 95f75573201..d5934c7d387 100644 --- a/tests/queries/0_stateless/00002_system_numbers.sql +++ b/tests/queries/0_stateless/00002_system_numbers.sql @@ -6,7 +6,7 @@ SELECT number FROM system.numbers WHERE number >= 5 LIMIT 2; SELECT * FROM system.numbers WHERE number == 7 LIMIT 1; SELECT number AS n FROM system.numbers WHERE number IN(8, 9) LIMIT 2; select number from system.numbers limit 0; -select x from system.numbers limit 1; -- { clientError 0 serverError 47 } +select x from system.numbers limit 1; -- { serverError UNKNOWN_IDENTIFIER } SELECT x, number FROM system.numbers LIMIT 1; -- { serverError 47 } SELECT * FROM system.number LIMIT 1; -- { serverError 60 } SELECT * FROM system LIMIT 1; -- { serverError 60 } diff --git a/tests/queries/0_stateless/00118_storage_join.sql b/tests/queries/0_stateless/00118_storage_join.sql index 17cfe437448..552e62afa9c 100644 --- a/tests/queries/0_stateless/00118_storage_join.sql +++ b/tests/queries/0_stateless/00118_storage_join.sql @@ -1,5 +1,3 @@ -SET allow_experimental_analyzer = 1; - DROP TABLE IF EXISTS t2; CREATE TABLE t2 (k UInt64, s String) ENGINE = Join(ANY, LEFT, k); @@ -18,6 +16,6 @@ SELECT k, js1.s, t2.s FROM (SELECT toUInt64(number / 3) AS k, sum(number) as s F SELECT k, js1.s, t2.s FROM (SELECT number AS k, number AS s FROM system.numbers LIMIT 10) js1 ANY LEFT JOIN t2 ON js1.k == t2.k ORDER BY k; SELECT k, t2.k, js1.s, t2.s FROM (SELECT number AS k, number AS s FROM system.numbers LIMIT 10) js1 ANY LEFT JOIN t2 ON js1.k == t2.k ORDER BY k; -SELECT k, js1.s, t2.s FROM (SELECT number AS k, number AS s FROM system.numbers LIMIT 10) js1 ANY LEFT JOIN t2 ON js1.k == t2.k OR js1.s == t2.k ORDER BY k; -- { serverError 264 } +SELECT k, js1.s, t2.s FROM (SELECT number AS k, number AS s FROM system.numbers LIMIT 10) js1 ANY LEFT JOIN t2 ON js1.k == t2.k OR js1.s == t2.k ORDER BY k; -- { serverError 48, 264 } DROP TABLE t2; diff --git a/tests/queries/0_stateless/00202_cross_join.reference b/tests/queries/0_stateless/00202_cross_join.reference index 122cf0a6e06..a8db281730a 100644 --- a/tests/queries/0_stateless/00202_cross_join.reference +++ b/tests/queries/0_stateless/00202_cross_join.reference @@ -13,3 +13,33 @@ 2 2 2 3 2 4 +0 0 +0 1 +0 2 +0 3 +0 4 +1 0 +1 1 +1 2 +1 3 +1 4 +2 0 +2 1 +2 2 +2 3 +2 4 +0 0 +0 1 +0 2 +0 3 +0 4 +1 0 +1 1 +1 2 +1 3 +1 4 +2 0 +2 1 +2 2 +2 3 +2 4 diff --git a/tests/queries/0_stateless/00202_cross_join.sql b/tests/queries/0_stateless/00202_cross_join.sql index ed435d90021..8d62c56b3f1 100644 --- a/tests/queries/0_stateless/00202_cross_join.sql +++ b/tests/queries/0_stateless/00202_cross_join.sql @@ -1 +1,7 @@ SELECT x, y FROM (SELECT number AS x FROM system.numbers LIMIT 3) js1 CROSS JOIN (SELECT number AS y FROM system.numbers LIMIT 5) js2; + +SET join_algorithm = 'auto'; +SELECT x, y FROM (SELECT number AS x FROM system.numbers LIMIT 3) js1 CROSS JOIN (SELECT number AS y FROM system.numbers LIMIT 5) js2; + +SET allow_experimental_analyzer = 1; +SELECT x, y FROM (SELECT number AS x FROM system.numbers LIMIT 3) js1 CROSS JOIN (SELECT number AS y FROM system.numbers LIMIT 5) js2; diff --git a/tests/queries/0_stateless/00378_json_quote_64bit_integers.reference b/tests/queries/0_stateless/00378_json_quote_64bit_integers.reference index 5174c13a9e0..b8d51e5d078 100644 --- a/tests/queries/0_stateless/00378_json_quote_64bit_integers.reference +++ b/tests/queries/0_stateless/00378_json_quote_64bit_integers.reference @@ -48,10 +48,10 @@ { "i0": "0", "u0": "0", - "ip": "9223372036854775807", - "in": "-9223372036854775808", - "up": "18446744073709551615", - "arr": ["0"], + "ip": "0", + "in": "0", + "up": "0", + "arr": [], "tuple": ["0","0"] }, @@ -119,7 +119,7 @@ ["0", "0", "9223372036854775807", "-9223372036854775808", "18446744073709551615", ["0"], ["0","0"]] ], - "totals": ["0", "0", "9223372036854775807", "-9223372036854775808", "18446744073709551615", ["0"], ["0","0"]], + "totals": ["0", "0", "0", "0", "0", [], ["0","0"]], "extremes": { @@ -180,10 +180,10 @@ { "i0": 0, "u0": 0, - "ip": 9223372036854775807, - "in": -9223372036854775808, - "up": 18446744073709551615, - "arr": [0], + "ip": 0, + "in": 0, + "up": 0, + "arr": [], "tuple": [0,0] }, @@ -251,7 +251,7 @@ [0, 0, 9223372036854775807, -9223372036854775808, 18446744073709551615, [0], [0,0]] ], - "totals": [0, 0, 9223372036854775807, -9223372036854775808, 18446744073709551615, [0], [0,0]], + "totals": [0, 0, 0, 0, 0, [], [0,0]], "extremes": { diff --git a/tests/queries/0_stateless/00378_json_quote_64bit_integers.sql b/tests/queries/0_stateless/00378_json_quote_64bit_integers.sql index 3a70b64bc86..e7b59bc3f7f 100644 --- a/tests/queries/0_stateless/00378_json_quote_64bit_integers.sql +++ b/tests/queries/0_stateless/00378_json_quote_64bit_integers.sql @@ -2,6 +2,7 @@ SET output_format_write_statistics = 0; SET extremes = 1; +SET allow_experimental_analyzer = 1; SET output_format_json_quote_64bit_integers = 1; SELECT toInt64(0) as i0, toUInt64(0) as u0, toInt64(9223372036854775807) as ip, toInt64(-9223372036854775808) as in, toUInt64(18446744073709551615) as up, [toInt64(0)] as arr, (toUInt64(0), toUInt64(0)) as tuple GROUP BY i0, u0, ip, in, up, arr, tuple WITH TOTALS FORMAT JSON; diff --git a/tests/queries/0_stateless/00386_has_column_in_table.sql b/tests/queries/0_stateless/00386_has_column_in_table.sql index d543bb42ca7..7347293e05b 100644 --- a/tests/queries/0_stateless/00386_has_column_in_table.sql +++ b/tests/queries/0_stateless/00386_has_column_in_table.sql @@ -21,11 +21,11 @@ SELECT hasColumnInTable('localhost', currentDatabase(), 'has_column_in_table', ' SELECT hasColumnInTable('system', 'one', ''); /* bad queries */ -SELECT hasColumnInTable('', '', ''); -- { serverError 60; } -SELECT hasColumnInTable('', 't', 'c'); -- { serverError 81; } -SELECT hasColumnInTable(currentDatabase(), '', 'c'); -- { serverError 60; } -SELECT hasColumnInTable('d', 't', 's'); -- { serverError 81; } -SELECT hasColumnInTable(currentDatabase(), 't', 's'); -- { serverError 60; } +SELECT hasColumnInTable('', '', ''); -- { serverError 60 } +SELECT hasColumnInTable('', 't', 'c'); -- { serverError 81 } +SELECT hasColumnInTable(currentDatabase(), '', 'c'); -- { serverError 60 } +SELECT hasColumnInTable('d', 't', 's'); -- { serverError 81 } +SELECT hasColumnInTable(currentDatabase(), 't', 's'); -- { serverError 60 } DROP TABLE has_column_in_table; diff --git a/tests/queries/0_stateless/00390_array_sort.sql b/tests/queries/0_stateless/00390_array_sort.sql index 1518761b166..bead64ff551 100644 --- a/tests/queries/0_stateless/00390_array_sort.sql +++ b/tests/queries/0_stateless/00390_array_sort.sql @@ -1,5 +1,3 @@ -SET allow_experimental_analyzer = 1; - SELECT [2, 1, 3] AS arr, arraySort(arr), arrayReverseSort(arr), arraySort(x -> -x, arr); SELECT materialize([2, 1, 3]) AS arr, arraySort(arr), arrayReverseSort(arr), arraySort(x -> -x, arr); @@ -53,4 +51,4 @@ SELECT arrayPartialSort(2, [1,2,3], [1]); -- { serverError ILLEGAL_TYPE_OF_ARGUM SELECT arrayPartialSort(2, [1,2,3], 3); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } SELECT arrayPartialSort(arraySort([1,2,3]), [1,2,3]); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } SELECT arrayMap(x -> range(x), [4, 1, 2, 3]) AS arr, 100 AS lim, arrayResize(arrayPartialSort(arrayPartialSort(lim, arr), arr), lim), arrayResize(arrayPartialReverseSort(lim, arr), lim), arrayResize(arrayPartialSort(x -> (-length(x)), lim, arr), lim); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } -SELECT arrayPartialReverseSort(arraySort((x, y) -> y, [NULL, NULL], [NULL, NULL]), arr), arrayMap(x -> toString(x), [257, -9223372036854775807, 2, -2147483648, 2147483648, NULL, 65536, -2147483648, 2, 65535]) AS arr, NULL, 100 AS lim, 65536, arrayResize(arrayPartialSort(x -> reverse(x), lim, arr), lim) GROUP BY [NULL, 1023, -2, NULL, 255, '0', NULL, 9223372036854775806] WITH ROLLUP; -- { serverError NO_COMMON_TYPE } +SELECT arrayPartialReverseSort(arraySort((x, y) -> y, [NULL, NULL], [NULL, NULL]), arr), arrayMap(x -> toString(x), [257, -9223372036854775807, 2, -2147483648, 2147483648, NULL, 65536, -2147483648, 2, 65535]) AS arr, NULL, 100 AS lim, 65536, arrayResize(arrayPartialSort(x -> reverse(x), lim, arr), lim) GROUP BY [NULL, 1023, -2, NULL, 255, '0', NULL, 9223372036854775806] WITH ROLLUP; -- { serverError ILLEGAL_TYPE_OF_ARGUMENT, NO_COMMON_TYPE } diff --git a/tests/queries/0_stateless/00445_join_nullable_keys.reference b/tests/queries/0_stateless/00445_join_nullable_keys.reference index afc8003910c..cc1c06d593b 100644 --- a/tests/queries/0_stateless/00445_join_nullable_keys.reference +++ b/tests/queries/0_stateless/00445_join_nullable_keys.reference @@ -22,13 +22,13 @@ 13 13 14 14 \N 8 -0 0 -0 2 -0 4 -0 6 -0 8 1 1 3 3 5 5 7 7 9 9 +\N 0 +\N 2 +\N 4 +\N 6 +\N 8 diff --git a/tests/queries/0_stateless/00445_join_nullable_keys.sql b/tests/queries/0_stateless/00445_join_nullable_keys.sql index a0453356e98..774594f90f3 100644 --- a/tests/queries/0_stateless/00445_join_nullable_keys.sql +++ b/tests/queries/0_stateless/00445_join_nullable_keys.sql @@ -1,3 +1,4 @@ +SET allow_experimental_analyzer = 1; SET join_use_nulls = 0; SET any_join_distinct_right_table_keys = 1; diff --git a/tests/queries/0_stateless/00534_filimonov.data b/tests/queries/0_stateless/00534_filimonov.data index 911a8e4d1f3..eb4500877e5 100644 --- a/tests/queries/0_stateless/00534_filimonov.data +++ b/tests/queries/0_stateless/00534_filimonov.data @@ -276,10 +276,14 @@ SELECT runningDifference(CAST( 0 AS Nullable(Int8))); SELECT runningDifference(CAST( 0 AS Nullable(Int16))); SELECT runningDifference(CAST( 0 AS Nullable(Int32))); SELECT runningDifference(CAST( 0 AS Nullable(Int64))); +SELECT runningDifference(CAST( 0 AS Nullable(Int128))); +SELECT runningDifference(CAST( 0 AS Nullable(Int256))); SELECT runningDifference(CAST( 0 AS Nullable(UInt8))); SELECT runningDifference(CAST( 0 AS Nullable(UInt16))); SELECT runningDifference(CAST( 0 AS Nullable(UInt32))); SELECT runningDifference(CAST( 0 AS Nullable(UInt64))); +SELECT runningDifference(CAST( 0 AS Nullable(UInt128))); +SELECT runningDifference(CAST( 0 AS Nullable(UInt256))); SELECT runningDifference(CAST( 0 AS Nullable(Float32))); SELECT runningDifference(CAST( 0 AS Nullable(Float64))); SELECT runningDifference(CAST( 0 AS Nullable(Date))); @@ -288,10 +292,14 @@ SELECT runningDifference(CAST(NULL AS Nullable(Int8))); SELECT runningDifference(CAST(NULL AS Nullable(Int16))); SELECT runningDifference(CAST(NULL AS Nullable(Int32))); SELECT runningDifference(CAST(NULL AS Nullable(Int64))); +SELECT runningDifference(CAST(NULL AS Nullable(Int128))); +SELECT runningDifference(CAST(NULL AS Nullable(Int256))); SELECT runningDifference(CAST(NULL AS Nullable(UInt8))); SELECT runningDifference(CAST(NULL AS Nullable(UInt16))); SELECT runningDifference(CAST(NULL AS Nullable(UInt32))); SELECT runningDifference(CAST(NULL AS Nullable(UInt64))); +SELECT runningDifference(CAST(NULL AS Nullable(UInt128))); +SELECT runningDifference(CAST(NULL AS Nullable(UInt256)); SELECT runningDifference(CAST(NULL AS Nullable(Float32))); SELECT runningDifference(CAST(NULL AS Nullable(Float64))); SELECT runningDifference(CAST(NULL AS Nullable(Date))); diff --git a/tests/queries/0_stateless/00534_functions_bad_arguments5.sh b/tests/queries/0_stateless/00534_functions_bad_arguments5.sh index 7b180870443..a8b0ce77677 100755 --- a/tests/queries/0_stateless/00534_functions_bad_arguments5.sh +++ b/tests/queries/0_stateless/00534_functions_bad_arguments5.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: no-tsan, no-debug +# Tags: no-tsan, no-debug, no-fasttest # Tag no-tsan: Too long for TSan # shellcheck disable=SC2016 diff --git a/tests/queries/0_stateless/00534_functions_bad_arguments7.sh b/tests/queries/0_stateless/00534_functions_bad_arguments7.sh index 8358d2b80d4..383e5a1b434 100755 --- a/tests/queries/0_stateless/00534_functions_bad_arguments7.sh +++ b/tests/queries/0_stateless/00534_functions_bad_arguments7.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: no-tsan, no-debug +# Tags: no-tsan, no-debug, no-fasttest # Tag no-tsan: Too long for TSan # shellcheck disable=SC2016 diff --git a/tests/queries/0_stateless/00561_storage_join.sql b/tests/queries/0_stateless/00561_storage_join.sql index ebdbe4dbc0a..6411628bbde 100644 --- a/tests/queries/0_stateless/00561_storage_join.sql +++ b/tests/queries/0_stateless/00561_storage_join.sql @@ -1,5 +1,3 @@ -SET allow_experimental_analyzer = 1; - drop table IF EXISTS joinbug; set allow_deprecated_syntax_for_merge_tree=1; @@ -38,7 +36,7 @@ SEMI LEFT JOIN joinbug_join using id2; SELECT * FROM ( SELECT toUInt32(11) AS id2 ) AS js1 SEMI LEFT JOIN joinbug_join USING (id2); -- can't convert right side in case on storage join -SELECT * FROM ( SELECT toInt64(11) AS id2 ) AS js1 SEMI LEFT JOIN joinbug_join USING (id2); -- { serverError 386 } +SELECT * FROM ( SELECT toInt64(11) AS id2 ) AS js1 SEMI LEFT JOIN joinbug_join USING (id2); -- { serverError 53, 386 } DROP TABLE joinbug; DROP TABLE joinbug_join; diff --git a/tests/queries/0_stateless/00621_regression_for_in_operator.reference b/tests/queries/0_stateless/00621_regression_for_in_operator.reference index 90f0a70449a..ab8bcf307eb 100644 --- a/tests/queries/0_stateless/00621_regression_for_in_operator.reference +++ b/tests/queries/0_stateless/00621_regression_for_in_operator.reference @@ -3,3 +3,43 @@ 2 2 2 +QUERY id: 0 + PROJECTION COLUMNS + count() UInt64 + PROJECTION + LIST id: 1, nodes: 1 + FUNCTION id: 2, function_name: count, function_type: aggregate, result_type: UInt64 + JOIN TREE + TABLE id: 3, table_name: default.regression_for_in_operator_view + WHERE + FUNCTION id: 4, function_name: in, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 5, nodes: 2 + COLUMN id: 6, column_name: g, result_type: String, source_id: 3 + CONSTANT id: 7, constant_value: Tuple_(\'5\', \'6\'), constant_value_type: Tuple(String, String) + SETTINGS allow_experimental_analyzer=1 +2 +2 +QUERY id: 0 + PROJECTION COLUMNS + count() UInt64 + PROJECTION + LIST id: 1, nodes: 1 + FUNCTION id: 2, function_name: count, function_type: aggregate, result_type: UInt64 + JOIN TREE + TABLE id: 3, table_name: default.regression_for_in_operator_view + WHERE + FUNCTION id: 4, function_name: or, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 5, nodes: 2 + FUNCTION id: 6, function_name: equals, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 7, nodes: 2 + COLUMN id: 8, column_name: g, result_type: String, source_id: 3 + CONSTANT id: 9, constant_value: \'5\', constant_value_type: String + FUNCTION id: 10, function_name: equals, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 11, nodes: 2 + COLUMN id: 8, column_name: g, result_type: String, source_id: 3 + CONSTANT id: 12, constant_value: \'6\', constant_value_type: String + SETTINGS allow_experimental_analyzer=1 diff --git a/tests/queries/0_stateless/00621_regression_for_in_operator.sql b/tests/queries/0_stateless/00621_regression_for_in_operator.sql index 273f930a90f..db1bcb4a39a 100644 --- a/tests/queries/0_stateless/00621_regression_for_in_operator.sql +++ b/tests/queries/0_stateless/00621_regression_for_in_operator.sql @@ -12,9 +12,13 @@ SELECT count() FROM regression_for_in_operator_view WHERE g IN ('5','6'); SET optimize_min_equality_disjunction_chain_length = 1; SELECT count() FROM regression_for_in_operator_view WHERE g = '5' OR g = '6'; +SELECT count() FROM regression_for_in_operator_view WHERE g = '5' OR g = '6' SETTINGS allow_experimental_analyzer = 1; +EXPLAIN QUERY TREE SELECT count() FROM regression_for_in_operator_view WHERE g = '5' OR g = '6' SETTINGS allow_experimental_analyzer = 1; SET optimize_min_equality_disjunction_chain_length = 3; SELECT count() FROM regression_for_in_operator_view WHERE g = '5' OR g = '6'; +SELECT count() FROM regression_for_in_operator_view WHERE g = '5' OR g = '6' SETTINGS allow_experimental_analyzer = 1; +EXPLAIN QUERY TREE SELECT count() FROM regression_for_in_operator_view WHERE g = '5' OR g = '6' SETTINGS allow_experimental_analyzer = 1; DROP TABLE regression_for_in_operator_view; DROP TABLE regression_for_in_operator; diff --git a/tests/queries/0_stateless/00636_partition_key_parts_pruning.sh b/tests/queries/0_stateless/00636_partition_key_parts_pruning.sh index fdaecd87f53..7ec4d99f028 100755 --- a/tests/queries/0_stateless/00636_partition_key_parts_pruning.sh +++ b/tests/queries/0_stateless/00636_partition_key_parts_pruning.sh @@ -7,7 +7,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) ${CLICKHOUSE_CLIENT} --query="SELECT '*** Single column partition key ***'" ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS single_col_partition_key" -${CLICKHOUSE_CLIENT} --query="CREATE TABLE single_col_partition_key(x UInt32) ENGINE MergeTree ORDER BY x PARTITION BY intDiv(x, 10)" +${CLICKHOUSE_CLIENT} --query="CREATE TABLE single_col_partition_key(x UInt32) ENGINE MergeTree ORDER BY x PARTITION BY intDiv(x, 10) SETTINGS index_granularity=4" ${CLICKHOUSE_CLIENT} --query="INSERT INTO single_col_partition_key VALUES (1), (2), (3), (4), (11), (12), (20)" diff --git a/tests/queries/0_stateless/00646_url_engine.python b/tests/queries/0_stateless/00646_url_engine.python index c955eaff643..5f3b7546dd5 100644 --- a/tests/queries/0_stateless/00646_url_engine.python +++ b/tests/queries/0_stateless/00646_url_engine.python @@ -82,6 +82,10 @@ class CSVHTTPServer(BaseHTTPRequestHandler): self.wfile.write((', '.join(row) + '\n').encode()) return + def do_HEAD(self): + self._set_headers() + return + def read_chunk(self): msg = '' while True: diff --git a/tests/queries/0_stateless/00653_running_difference.reference b/tests/queries/0_stateless/00653_running_difference.reference index e2833e0bb3e..624ce92ce0f 100644 --- a/tests/queries/0_stateless/00653_running_difference.reference +++ b/tests/queries/0_stateless/00653_running_difference.reference @@ -19,6 +19,30 @@ \N \N 2 +- +0 +1 +4 +5 +170141183460469231731687303715884105717 +- +0 +1 +4 +5 +170141183460469231731687303715884105718 +- +0 +1 +4 +5 +170141183460469231731687303715884105717 +- +0 +1 +4 +5 +170141183460469231731687303715884105718 --Date Difference-- \N \N diff --git a/tests/queries/0_stateless/00653_running_difference.sql b/tests/queries/0_stateless/00653_running_difference.sql index f2b4a7300b2..1f18cfc42a7 100644 --- a/tests/queries/0_stateless/00653_running_difference.sql +++ b/tests/queries/0_stateless/00653_running_difference.sql @@ -5,6 +5,14 @@ select '-'; select runningDifference(x) from (select arrayJoin([Null, 1]) as x); select '-'; select runningDifference(x) from (select arrayJoin([Null, Null, 1, 3, Null, Null, 5]) as x); +select '-'; +select runningDifference(x) from (select arrayJoin([0, 1, 5, 10, 170141183460469231731687303715884105727]::Array(UInt128)) as x); +select '-'; +select runningDifference(x) from (select arrayJoin([0, 1, 5, 10, 170141183460469231731687303715884105728]::Array(UInt256)) as x); +select '-'; +select runningDifference(x) from (select arrayJoin([0, 1, 5, 10, 170141183460469231731687303715884105727]::Array(Int128)) as x); +select '-'; +select runningDifference(x) from (select arrayJoin([0, 1, 5, 10, 170141183460469231731687303715884105728]::Array(Int256)) as x); select '--Date Difference--'; select runningDifference(x) from (select arrayJoin([Null, Null, toDate('1970-1-1'), toDate('1970-12-31'), Null, Null, toDate('2010-8-9')]) as x); select '-'; diff --git a/tests/queries/0_stateless/00718_format_datetime.sql b/tests/queries/0_stateless/00718_format_datetime.sql index 74ec03d83d3..3f8c927dfe7 100644 --- a/tests/queries/0_stateless/00718_format_datetime.sql +++ b/tests/queries/0_stateless/00718_format_datetime.sql @@ -1,14 +1,14 @@ SET send_logs_level = 'fatal'; -SELECT formatDateTime(); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH (42) } -SELECT formatDateTime('not a datetime', 'IGNORED'); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT (43) } -SELECT formatDateTime(now(), now()); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT (43) } -SELECT formatDateTime(now(), 'good format pattern', now()); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT (43) } -SELECT formatDateTime(now(), 'unescaped %'); -- { serverError BAD_ARGUMENTS (36) } -SELECT formatDateTime(toDateTime('2018-01-02 22:33:44'), '%U'); -- { serverError NOT_IMPLEMENTED (48) } -SELECT formatDateTime(toDateTime('2018-01-02 22:33:44'), '%v'); -- { serverError NOT_IMPLEMENTED (48) } -SELECT formatDateTime(toDateTime('2018-01-02 22:33:44'), '%x'); -- { serverError NOT_IMPLEMENTED (48) } -SELECT formatDateTime(toDateTime('2018-01-02 22:33:44'), '%X'); -- { serverError NOT_IMPLEMENTED (48) } +SELECT formatDateTime(); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +SELECT formatDateTime('not a datetime', 'IGNORED'); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +SELECT formatDateTime(now(), now()); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +SELECT formatDateTime(now(), 'good format pattern', now()); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +SELECT formatDateTime(now(), 'unescaped %'); -- { serverError BAD_ARGUMENTS } +SELECT formatDateTime(toDateTime('2018-01-02 22:33:44'), '%U'); -- { serverError NOT_IMPLEMENTED } +SELECT formatDateTime(toDateTime('2018-01-02 22:33:44'), '%v'); -- { serverError NOT_IMPLEMENTED } +SELECT formatDateTime(toDateTime('2018-01-02 22:33:44'), '%x'); -- { serverError NOT_IMPLEMENTED } +SELECT formatDateTime(toDateTime('2018-01-02 22:33:44'), '%X'); -- { serverError NOT_IMPLEMENTED } SELECT formatDateTime(toDateTime('2018-01-02 22:33:44'), '%a'), formatDateTime(toDate32('2018-01-02'), '%a'); SELECT formatDateTime(toDateTime('2018-01-02 22:33:44'), '%b'), formatDateTime(toDate32('2018-01-02'), '%b'); diff --git a/tests/queries/0_stateless/00719_parallel_ddl_table.sh b/tests/queries/0_stateless/00719_parallel_ddl_table.sh index 2a542ea21f6..fdc994aec33 100755 --- a/tests/queries/0_stateless/00719_parallel_ddl_table.sh +++ b/tests/queries/0_stateless/00719_parallel_ddl_table.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# Tags: no-fasttest set -e CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) diff --git a/tests/queries/0_stateless/00722_inner_join.reference b/tests/queries/0_stateless/00722_inner_join.reference index 86c07e6e84e..b5e8a77a20d 100644 --- a/tests/queries/0_stateless/00722_inner_join.reference +++ b/tests/queries/0_stateless/00722_inner_join.reference @@ -16,24 +16,24 @@ ┌─x──────┬─name─┐ │ system │ one │ └────────┴──────┘ -┌─database─┬─t.name─┐ -│ system │ one │ -└──────────┴────────┘ +┌─database─┬─name─┐ +│ system │ one │ +└──────────┴──────┘ ┌─db.x───┬─name─┐ │ system │ one │ └────────┴──────┘ -┌─db.name─┬─t.name─┐ -│ system │ one │ -└─────────┴────────┘ -┌─db.name─┬─t.name─┐ -│ system │ one │ -└─────────┴────────┘ -┌─t.database─┬─t.name─┐ -│ system │ one │ -└────────────┴────────┘ -┌─database─┬─t.name─┐ -│ system │ one │ -└──────────┴────────┘ +┌─db.name─┬─name─┐ +│ system │ one │ +└─────────┴──────┘ +┌─db.name─┬─name─┐ +│ system │ one │ +└─────────┴──────┘ +┌─database─┬─name─┐ +│ system │ one │ +└──────────┴──────┘ +┌─database─┬─name─┐ +│ system │ one │ +└──────────┴──────┘ 2 2 2 diff --git a/tests/queries/0_stateless/00722_inner_join.sql b/tests/queries/0_stateless/00722_inner_join.sql index 75ef40ff2b7..0d5a543b99d 100644 --- a/tests/queries/0_stateless/00722_inner_join.sql +++ b/tests/queries/0_stateless/00722_inner_join.sql @@ -1,3 +1,5 @@ +-- Tags: no-parallel + SET allow_experimental_analyzer = 1; DROP TABLE IF EXISTS one; diff --git a/tests/queries/0_stateless/00736_disjunction_optimisation.reference b/tests/queries/0_stateless/00736_disjunction_optimisation.reference index afd698b425e..84477a64057 100644 --- a/tests/queries/0_stateless/00736_disjunction_optimisation.reference +++ b/tests/queries/0_stateless/00736_disjunction_optimisation.reference @@ -25,6 +25,81 @@ 3 21 3 22 3 23 +QUERY id: 0 + PROJECTION COLUMNS + k UInt64 + s UInt64 + PROJECTION + LIST id: 1, nodes: 2 + COLUMN id: 2, column_name: k, result_type: UInt64, source_id: 3 + COLUMN id: 4, column_name: s, result_type: UInt64, source_id: 3 + JOIN TREE + TABLE id: 3, table_name: default.bug + WHERE + FUNCTION id: 5, function_name: and, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 6, nodes: 2 + FUNCTION id: 7, function_name: in, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 8, nodes: 2 + COLUMN id: 9, column_name: k, result_type: UInt64, source_id: 3 + CONSTANT id: 10, constant_value: Tuple_(UInt64_1, UInt64_2, UInt64_3), constant_value_type: Tuple(UInt8, UInt8, UInt8) + FUNCTION id: 11, function_name: in, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 12, nodes: 2 + COLUMN id: 13, column_name: s, result_type: UInt64, source_id: 3 + CONSTANT id: 14, constant_value: Tuple_(UInt64_21, UInt64_22, UInt64_23), constant_value_type: Tuple(UInt8, UInt8, UInt8) + SETTINGS allow_experimental_analyzer=1 +1 21 +1 22 +1 23 +2 21 +2 22 +2 23 +3 21 +3 22 +3 23 +1 21 +1 22 +1 23 +2 21 +2 22 +2 23 +3 21 +3 22 +3 23 +QUERY id: 0 + PROJECTION COLUMNS + k UInt64 + s UInt64 + PROJECTION + LIST id: 1, nodes: 2 + COLUMN id: 2, column_name: k, result_type: UInt64, source_id: 3 + COLUMN id: 4, column_name: s, result_type: UInt64, source_id: 3 + JOIN TREE + QUERY id: 3, is_subquery: 1 + PROJECTION COLUMNS + k UInt64 + s UInt64 + PROJECTION + LIST id: 5, nodes: 2 + COLUMN id: 6, column_name: k, result_type: UInt64, source_id: 7 + COLUMN id: 8, column_name: s, result_type: UInt64, source_id: 7 + JOIN TREE + TABLE id: 7, table_name: default.bug + WHERE + FUNCTION id: 9, function_name: in, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 10, nodes: 2 + COLUMN id: 11, column_name: k, result_type: UInt64, source_id: 7 + CONSTANT id: 12, constant_value: Tuple_(UInt64_1, UInt64_2, UInt64_3), constant_value_type: Tuple(UInt8, UInt8, UInt8) + WHERE + FUNCTION id: 13, function_name: in, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 14, nodes: 2 + COLUMN id: 15, column_name: s, result_type: UInt64, source_id: 3 + CONSTANT id: 16, constant_value: Tuple_(UInt64_21, UInt64_22, UInt64_23), constant_value_type: Tuple(UInt8, UInt8, UInt8) + SETTINGS allow_experimental_analyzer=1 1 1 21 1 1 1 1 1 22 0 1 1 1 1 23 0 0 1 @@ -34,42 +109,6 @@ 3 1 21 1 1 1 3 1 22 0 1 1 3 1 23 0 0 1 -21 1 -22 1 -23 1 -21 1 -22 1 -23 1 -21 1 -22 1 -23 1 -1 21 -1 22 -1 23 -2 21 -2 22 -2 23 -3 21 -3 22 -3 23 -1 21 -1 22 -1 23 -2 21 -2 22 -2 23 -3 21 -3 22 -3 23 -1 21 -1 22 -1 23 -2 21 -2 22 -2 23 -3 21 -3 22 -3 23 1 1 21 1 1 1 1 1 22 0 1 1 1 1 23 0 0 1 @@ -79,6 +118,41 @@ 3 1 21 1 1 1 3 1 22 0 1 1 3 1 23 0 0 1 +QUERY id: 0 + PROJECTION COLUMNS + k UInt64 + or(equals(k, 1), equals(k, 2), equals(k, 3)) UInt8 + s UInt64 + equals(s, 21) UInt8 + or(equals(s, 21), equals(s, 22)) UInt8 + or(equals(s, 21), equals(s, 22), equals(s, 23)) UInt8 + PROJECTION + LIST id: 1, nodes: 6 + COLUMN id: 2, column_name: k, result_type: UInt64, source_id: 3 + FUNCTION id: 4, function_name: in, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 5, nodes: 2 + COLUMN id: 2, column_name: k, result_type: UInt64, source_id: 3 + CONSTANT id: 6, constant_value: Tuple_(UInt64_1, UInt64_2, UInt64_3), constant_value_type: Tuple(UInt8, UInt8, UInt8) + COLUMN id: 7, column_name: s, result_type: UInt64, source_id: 3 + FUNCTION id: 8, function_name: equals, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 9, nodes: 2 + COLUMN id: 7, column_name: s, result_type: UInt64, source_id: 3 + CONSTANT id: 10, constant_value: UInt64_21, constant_value_type: UInt8 + FUNCTION id: 11, function_name: in, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 12, nodes: 2 + COLUMN id: 7, column_name: s, result_type: UInt64, source_id: 3 + CONSTANT id: 13, constant_value: Tuple_(UInt64_21, UInt64_22), constant_value_type: Tuple(UInt8, UInt8) + FUNCTION id: 14, function_name: in, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 15, nodes: 2 + COLUMN id: 7, column_name: s, result_type: UInt64, source_id: 3 + CONSTANT id: 16, constant_value: Tuple_(UInt64_21, UInt64_22, UInt64_23), constant_value_type: Tuple(UInt8, UInt8, UInt8) + JOIN TREE + TABLE id: 3, table_name: default.bug + SETTINGS allow_experimental_analyzer=1 21 1 22 1 23 1 @@ -88,3 +162,256 @@ 21 1 22 1 23 1 +21 1 +22 1 +23 1 +21 1 +22 1 +23 1 +21 1 +22 1 +23 1 +QUERY id: 0 + PROJECTION COLUMNS + s UInt64 + or(equals(s, 21), equals(s, 22), equals(s, 23)) UInt8 + PROJECTION + LIST id: 1, nodes: 2 + COLUMN id: 2, column_name: s, result_type: UInt64, source_id: 3 + FUNCTION id: 4, function_name: in, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 5, nodes: 2 + COLUMN id: 2, column_name: s, result_type: UInt64, source_id: 3 + CONSTANT id: 6, constant_value: Tuple_(UInt64_21, UInt64_22, UInt64_23), constant_value_type: Tuple(UInt8, UInt8, UInt8) + JOIN TREE + TABLE id: 3, table_name: default.bug + SETTINGS allow_experimental_analyzer=1 +1 21 +1 22 +1 23 +2 21 +2 22 +2 23 +3 21 +3 22 +3 23 +1 21 +1 22 +1 23 +2 21 +2 22 +2 23 +3 21 +3 22 +3 23 +1 21 +1 22 +1 23 +2 21 +2 22 +2 23 +3 21 +3 22 +3 23 +QUERY id: 0 + PROJECTION COLUMNS + k UInt64 + s UInt64 + PROJECTION + LIST id: 1, nodes: 2 + COLUMN id: 2, column_name: k, result_type: UInt64, source_id: 3 + COLUMN id: 4, column_name: s, result_type: UInt64, source_id: 3 + JOIN TREE + TABLE id: 3, table_name: default.bug + WHERE + FUNCTION id: 5, function_name: and, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 6, nodes: 2 + FUNCTION id: 7, function_name: in, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 8, nodes: 2 + COLUMN id: 9, column_name: k, result_type: UInt64, source_id: 3 + CONSTANT id: 10, constant_value: Tuple_(UInt64_1, UInt64_2, UInt64_3), constant_value_type: Tuple(UInt8, UInt8, UInt8) + FUNCTION id: 11, function_name: in, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 12, nodes: 2 + COLUMN id: 13, column_name: s, result_type: UInt64, source_id: 3 + CONSTANT id: 14, constant_value: Tuple_(UInt64_21, UInt64_22, UInt64_23), constant_value_type: Tuple(UInt8, UInt8, UInt8) + SETTINGS allow_experimental_analyzer=1 +1 21 +1 22 +1 23 +2 21 +2 22 +2 23 +3 21 +3 22 +3 23 +1 21 +1 22 +1 23 +2 21 +2 22 +2 23 +3 21 +3 22 +3 23 +QUERY id: 0 + PROJECTION COLUMNS + k UInt64 + s UInt64 + PROJECTION + LIST id: 1, nodes: 2 + COLUMN id: 2, column_name: k, result_type: UInt64, source_id: 3 + COLUMN id: 4, column_name: s, result_type: UInt64, source_id: 3 + JOIN TREE + QUERY id: 3, is_subquery: 1 + PROJECTION COLUMNS + k UInt64 + s UInt64 + PROJECTION + LIST id: 5, nodes: 2 + COLUMN id: 6, column_name: k, result_type: UInt64, source_id: 7 + COLUMN id: 8, column_name: s, result_type: UInt64, source_id: 7 + JOIN TREE + TABLE id: 7, table_name: default.bug + WHERE + FUNCTION id: 9, function_name: in, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 10, nodes: 2 + COLUMN id: 11, column_name: k, result_type: UInt64, source_id: 7 + CONSTANT id: 12, constant_value: Tuple_(UInt64_1, UInt64_2, UInt64_3), constant_value_type: Tuple(UInt8, UInt8, UInt8) + WHERE + FUNCTION id: 13, function_name: in, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 14, nodes: 2 + COLUMN id: 15, column_name: s, result_type: UInt64, source_id: 3 + CONSTANT id: 16, constant_value: Tuple_(UInt64_21, UInt64_22, UInt64_23), constant_value_type: Tuple(UInt8, UInt8, UInt8) + SETTINGS allow_experimental_analyzer=1 +1 1 21 1 1 1 +1 1 22 0 1 1 +1 1 23 0 0 1 +2 1 21 1 1 1 +2 1 22 0 1 1 +2 1 23 0 0 1 +3 1 21 1 1 1 +3 1 22 0 1 1 +3 1 23 0 0 1 +1 1 21 1 1 1 +1 1 22 0 1 1 +1 1 23 0 0 1 +2 1 21 1 1 1 +2 1 22 0 1 1 +2 1 23 0 0 1 +3 1 21 1 1 1 +3 1 22 0 1 1 +3 1 23 0 0 1 +QUERY id: 0 + PROJECTION COLUMNS + k UInt64 + or(equals(k, 1), equals(k, 2), equals(k, 3)) UInt8 + s UInt64 + equals(s, 21) UInt8 + or(equals(s, 21), equals(s, 22)) UInt8 + or(equals(s, 21), equals(s, 22), equals(s, 23)) UInt8 + PROJECTION + LIST id: 1, nodes: 6 + COLUMN id: 2, column_name: k, result_type: UInt64, source_id: 3 + FUNCTION id: 4, function_name: in, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 5, nodes: 2 + COLUMN id: 2, column_name: k, result_type: UInt64, source_id: 3 + CONSTANT id: 6, constant_value: Tuple_(UInt64_1, UInt64_2, UInt64_3), constant_value_type: Tuple(UInt8, UInt8, UInt8) + COLUMN id: 7, column_name: s, result_type: UInt64, source_id: 3 + FUNCTION id: 8, function_name: equals, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 9, nodes: 2 + COLUMN id: 7, column_name: s, result_type: UInt64, source_id: 3 + CONSTANT id: 10, constant_value: UInt64_21, constant_value_type: UInt8 + FUNCTION id: 11, function_name: or, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 12, nodes: 2 + FUNCTION id: 13, function_name: equals, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 14, nodes: 2 + COLUMN id: 7, column_name: s, result_type: UInt64, source_id: 3 + CONSTANT id: 15, constant_value: UInt64_21, constant_value_type: UInt8 + FUNCTION id: 16, function_name: equals, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 17, nodes: 2 + COLUMN id: 7, column_name: s, result_type: UInt64, source_id: 3 + CONSTANT id: 18, constant_value: UInt64_22, constant_value_type: UInt8 + FUNCTION id: 19, function_name: in, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 20, nodes: 2 + COLUMN id: 7, column_name: s, result_type: UInt64, source_id: 3 + CONSTANT id: 21, constant_value: Tuple_(UInt64_21, UInt64_22, UInt64_23), constant_value_type: Tuple(UInt8, UInt8, UInt8) + JOIN TREE + TABLE id: 3, table_name: default.bug + SETTINGS allow_experimental_analyzer=1 +21 1 +22 1 +23 1 +21 1 +22 1 +23 1 +21 1 +22 1 +23 1 +21 1 +22 1 +23 1 +21 1 +22 1 +23 1 +21 1 +22 1 +23 1 +QUERY id: 0 + PROJECTION COLUMNS + s UInt64 + or(equals(s, 21), equals(s, 22), equals(s, 23)) UInt8 + PROJECTION + LIST id: 1, nodes: 2 + COLUMN id: 2, column_name: s, result_type: UInt64, source_id: 3 + FUNCTION id: 4, function_name: in, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 5, nodes: 2 + COLUMN id: 2, column_name: s, result_type: UInt64, source_id: 3 + CONSTANT id: 6, constant_value: Tuple_(UInt64_21, UInt64_22, UInt64_23), constant_value_type: Tuple(UInt8, UInt8, UInt8) + JOIN TREE + TABLE id: 3, table_name: default.bug + SETTINGS allow_experimental_analyzer=1 +21 1 +22 1 +23 1 +21 1 +22 1 +23 1 +21 1 +22 1 +23 1 +21 1 +22 1 +23 1 +21 1 +22 1 +23 1 +21 1 +22 1 +23 1 +QUERY id: 0 + PROJECTION COLUMNS + s UInt64 + or(equals(s, 21), equals(22, s), equals(23, s)) UInt8 + PROJECTION + LIST id: 1, nodes: 2 + COLUMN id: 2, column_name: s, result_type: UInt64, source_id: 3 + FUNCTION id: 4, function_name: in, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 5, nodes: 2 + COLUMN id: 2, column_name: s, result_type: UInt64, source_id: 3 + CONSTANT id: 6, constant_value: Tuple_(UInt64_21, UInt64_22, UInt64_23), constant_value_type: Tuple(UInt8, UInt8, UInt8) + JOIN TREE + TABLE id: 3, table_name: default.bug + SETTINGS allow_experimental_analyzer=1 diff --git a/tests/queries/0_stateless/00736_disjunction_optimisation.sql b/tests/queries/0_stateless/00736_disjunction_optimisation.sql index 700221ef7f0..e5bfc81f7ae 100644 --- a/tests/queries/0_stateless/00736_disjunction_optimisation.sql +++ b/tests/queries/0_stateless/00736_disjunction_optimisation.sql @@ -5,17 +5,45 @@ insert into bug values(1,21),(1,22),(1,23),(2,21),(2,22),(2,23),(3,21),(3,22),(3 set optimize_min_equality_disjunction_chain_length = 2; select * from bug; + select * from bug where (k =1 or k=2 or k =3) and (s=21 or s=22 or s=23); +select * from bug where (k =1 or k=2 or k =3) and (s=21 or s=22 or s=23) SETTINGS allow_experimental_analyzer = 1; +explain query tree select * from bug where (k =1 or k=2 or k =3) and (s=21 or s=22 or s=23) SETTINGS allow_experimental_analyzer = 1;; + select * from (select * from bug where k=1 or k=2 or k=3) where (s=21 or s=22 or s=23); +select * from (select * from bug where k=1 or k=2 or k=3) where (s=21 or s=22 or s=23) SETTINGS allow_experimental_analyzer = 1;; +explain query tree select * from (select * from bug where k=1 or k=2 or k=3) where (s=21 or s=22 or s=23) SETTINGS allow_experimental_analyzer = 1;; + select k, (k=1 or k=2 or k=3), s, (s=21), (s=21 or s=22), (s=21 or s=22 or s=23) from bug; +select k, (k=1 or k=2 or k=3), s, (s=21), (s=21 or s=22), (s=21 or s=22 or s=23) from bug SETTINGS allow_experimental_analyzer = 1;; +explain query tree select k, (k=1 or k=2 or k=3), s, (s=21), (s=21 or s=22), (s=21 or s=22 or s=23) from bug SETTINGS allow_experimental_analyzer = 1;; + select s, (s=21 or s=22 or s=23) from bug; +select s, (s=21 or s=22 or s=23) from bug SETTINGS allow_experimental_analyzer = 1;; +explain query tree select s, (s=21 or s=22 or s=23) from bug SETTINGS allow_experimental_analyzer = 1;; set optimize_min_equality_disjunction_chain_length = 3; select * from bug; + select * from bug where (k =1 or k=2 or k =3) and (s=21 or s=22 or s=23); +select * from bug where (k =1 or k=2 or k =3) and (s=21 or s=22 or s=23) SETTINGS allow_experimental_analyzer = 1; +explain query tree select * from bug where (k =1 or k=2 or k =3) and (s=21 or s=22 or s=23) SETTINGS allow_experimental_analyzer = 1;; + select * from (select * from bug where k=1 or k=2 or k=3) where (s=21 or s=22 or s=23); +select * from (select * from bug where k=1 or k=2 or k=3) where (s=21 or s=22 or s=23) SETTINGS allow_experimental_analyzer = 1;; +explain query tree select * from (select * from bug where k=1 or k=2 or k=3) where (s=21 or s=22 or s=23) SETTINGS allow_experimental_analyzer = 1;; + select k, (k=1 or k=2 or k=3), s, (s=21), (s=21 or s=22), (s=21 or s=22 or s=23) from bug; +select k, (k=1 or k=2 or k=3), s, (s=21), (s=21 or s=22), (s=21 or s=22 or s=23) from bug SETTINGS allow_experimental_analyzer = 1;; +explain query tree select k, (k=1 or k=2 or k=3), s, (s=21), (s=21 or s=22), (s=21 or s=22 or s=23) from bug SETTINGS allow_experimental_analyzer = 1;; + select s, (s=21 or s=22 or s=23) from bug; +select s, (s=21 or s=22 or s=23) from bug SETTINGS allow_experimental_analyzer = 1;; +explain query tree select s, (s=21 or s=22 or s=23) from bug SETTINGS allow_experimental_analyzer = 1;; + +select s, (s=21 or 22=s or 23=s) from bug; +select s, (s=21 or 22=s or 23=s) from bug SETTINGS allow_experimental_analyzer = 1;; +explain query tree select s, (s=21 or 22=s or 23=s) from bug SETTINGS allow_experimental_analyzer = 1;; DROP TABLE bug; diff --git a/tests/queries/0_stateless/00746_sql_fuzzy.sh b/tests/queries/0_stateless/00746_sql_fuzzy.sh index b534b1820ba..c0741beea12 100755 --- a/tests/queries/0_stateless/00746_sql_fuzzy.sh +++ b/tests/queries/0_stateless/00746_sql_fuzzy.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# Tags: no-fasttest CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh diff --git a/tests/queries/0_stateless/00834_limit_with_constant_expressions.sql b/tests/queries/0_stateless/00834_limit_with_constant_expressions.sql index 1d3429436e7..54486634130 100644 --- a/tests/queries/0_stateless/00834_limit_with_constant_expressions.sql +++ b/tests/queries/0_stateless/00834_limit_with_constant_expressions.sql @@ -1,5 +1,3 @@ -SET allow_experimental_analyzer = 1; - SELECT number FROM numbers(10) LIMIT 0 + 1; SELECT number FROM numbers(10) LIMIT 1 - 1; SELECT number FROM numbers(10) LIMIT 2 - 1; @@ -11,7 +9,7 @@ SELECT number FROM numbers(10) LIMIT now(); -- { serverError 440 } SELECT number FROM numbers(10) LIMIT today(); -- { serverError 440 } SELECT number FROM numbers(10) LIMIT toUInt8('1'); SELECT number FROM numbers(10) LIMIT toFloat32('1'); -SELECT number FROM numbers(10) LIMIT rand(); -- { serverError 440 } +SELECT number FROM numbers(10) LIMIT rand(); -- { serverError 36, 440 } SELECT count() <= 1 FROM (SELECT number FROM numbers(10) LIMIT randConstant() % 2); diff --git a/tests/queries/0_stateless/00848_join_use_nulls_segfault.reference b/tests/queries/0_stateless/00848_join_use_nulls_segfault.reference index 6bfe0db1448..43f48089b06 100644 --- a/tests/queries/0_stateless/00848_join_use_nulls_segfault.reference +++ b/tests/queries/0_stateless/00848_join_use_nulls_segfault.reference @@ -10,13 +10,13 @@ l \N \N String Nullable(String) \N \N \N \N using -l \N String Nullable(String) - \N String Nullable(String) -l \N String Nullable(String) +l \N Nullable(String) Nullable(String) +l \N Nullable(String) Nullable(String) +\N \N Nullable(String) Nullable(String) +\N \N Nullable(String) Nullable(String) +l \N Nullable(String) Nullable(String) +l \N Nullable(String) Nullable(String) \N \N Nullable(String) Nullable(String) -l \N String Nullable(String) - \N String Nullable(String) -l \N String Nullable(String) \N \N Nullable(String) Nullable(String) \N \N \N \N @@ -32,13 +32,13 @@ l \N \N Nullable(String) Nullable(String) \N \N \N \N using + join_use_nulls -l \N String Nullable(String) l \N Nullable(String) Nullable(String) -\N \N Nullable(String) Nullable(String) -\N \N Nullable(String) Nullable(String) -l \N String Nullable(String) l \N Nullable(String) Nullable(String) -\N \N Nullable(String) Nullable(String) -\N \N Nullable(String) Nullable(String) +r \N Nullable(String) Nullable(String) +r \N Nullable(String) Nullable(String) +l \N Nullable(String) Nullable(String) +l \N Nullable(String) Nullable(String) +r \N Nullable(String) Nullable(String) +r \N Nullable(String) Nullable(String) \N \N \N \N diff --git a/tests/queries/0_stateless/00848_join_use_nulls_segfault.sql b/tests/queries/0_stateless/00848_join_use_nulls_segfault.sql index 57eca0eb9e0..2f6cca0284c 100644 --- a/tests/queries/0_stateless/00848_join_use_nulls_segfault.sql +++ b/tests/queries/0_stateless/00848_join_use_nulls_segfault.sql @@ -1,4 +1,5 @@ SET any_join_distinct_right_table_keys = 1; +SET allow_experimental_analyzer = 1; DROP TABLE IF EXISTS t1_00848; DROP TABLE IF EXISTS t2_00848; @@ -53,16 +54,16 @@ SELECT t3.id = 'l', t3.not_id = 'l' FROM t1_00848 t1 LEFT JOIN t3_00848 t3 ON t1 SELECT 'using + join_use_nulls'; -SELECT *, toTypeName(t1.id), toTypeName(t3.id) FROM t1_00848 t1 ANY LEFT JOIN t3_00848 t3 USING(id) ORDER BY t1.id, t3.id; -SELECT *, toTypeName(t1.id), toTypeName(t3.id) FROM t1_00848 t1 ANY FULL JOIN t3_00848 t3 USING(id) ORDER BY t1.id, t3.id; -SELECT *, toTypeName(t2.id), toTypeName(t3.id) FROM t2_00848 t2 ANY FULL JOIN t3_00848 t3 USING(id) ORDER BY t2.id, t3.id; +SELECT *, toTypeName(t1.id), toTypeName(t3.id) FROM t1_00848 t1 ANY LEFT JOIN t3_00848 t3 USING(id) ORDER BY id; +SELECT *, toTypeName(t1.id), toTypeName(t3.id) FROM t1_00848 t1 ANY FULL JOIN t3_00848 t3 USING(id) ORDER BY id; +SELECT *, toTypeName(t2.id), toTypeName(t3.id) FROM t2_00848 t2 ANY FULL JOIN t3_00848 t3 USING(id) ORDER BY id; -SELECT *, toTypeName(t1.id), toTypeName(t3.id) FROM t1_00848 t1 LEFT JOIN t3_00848 t3 USING(id) ORDER BY t1.id, t3.id; -SELECT *, toTypeName(t1.id), toTypeName(t3.id) FROM t1_00848 t1 FULL JOIN t3_00848 t3 USING(id) ORDER BY t1.id, t3.id; -SELECT *, toTypeName(t2.id), toTypeName(t3.id) FROM t2_00848 t2 FULL JOIN t3_00848 t3 USING(id) ORDER BY t2.id, t3.id; +SELECT *, toTypeName(t1.id), toTypeName(t3.id) FROM t1_00848 t1 LEFT JOIN t3_00848 t3 USING(id) ORDER BY id; +SELECT *, toTypeName(t1.id), toTypeName(t3.id) FROM t1_00848 t1 FULL JOIN t3_00848 t3 USING(id) ORDER BY id; +SELECT *, toTypeName(t2.id), toTypeName(t3.id) FROM t2_00848 t2 FULL JOIN t3_00848 t3 USING(id) ORDER BY id; -SELECT t3.id = 'l', t3.not_id = 'l' FROM t1_00848 t1 ANY LEFT JOIN t3_00848 t3 USING(id) ORDER BY t1.id, t3.id; -SELECT t3.id = 'l', t3.not_id = 'l' FROM t1_00848 t1 LEFT JOIN t3_00848 t3 USING(id) ORDER BY t1.id, t3.id; +SELECT t3.id = 'l', t3.not_id = 'l' FROM t1_00848 t1 ANY LEFT JOIN t3_00848 t3 USING(id) ORDER BY id; +SELECT t3.id = 'l', t3.not_id = 'l' FROM t1_00848 t1 LEFT JOIN t3_00848 t3 USING(id) ORDER BY id; DROP TABLE t1_00848; DROP TABLE t2_00848; diff --git a/tests/queries/0_stateless/00853_join_with_nulls_crash.reference b/tests/queries/0_stateless/00853_join_with_nulls_crash.reference index 459b73acdbf..5df14d02d5e 100644 --- a/tests/queries/0_stateless/00853_join_with_nulls_crash.reference +++ b/tests/queries/0_stateless/00853_join_with_nulls_crash.reference @@ -15,5 +15,5 @@ bar bar 1 2 String Nullable(String) \N 0 1 String Nullable(String) foo 2 0 String bar 1 2 String -test 0 1 String + 0 1 String 0 1 String diff --git a/tests/queries/0_stateless/00853_join_with_nulls_crash.sql b/tests/queries/0_stateless/00853_join_with_nulls_crash.sql index c63c2d99cba..b620b8a7902 100644 --- a/tests/queries/0_stateless/00853_join_with_nulls_crash.sql +++ b/tests/queries/0_stateless/00853_join_with_nulls_crash.sql @@ -27,7 +27,7 @@ SELECT s1.other, s2.other, count_a, count_b, toTypeName(s1.other), toTypeName(s2 ( SELECT other, count() AS count_a FROM table_a GROUP BY other ) s1 ALL FULL JOIN ( SELECT other, count() AS count_b FROM table_b GROUP BY other ) s2 -USING other +ON s1.other = s2.other ORDER BY s2.other DESC, count_a, s1.other; SELECT s1.something, s2.something, count_a, count_b, toTypeName(s1.something), toTypeName(s2.something) FROM @@ -41,7 +41,7 @@ SELECT s1.something, s2.something, count_a, count_b, toTypeName(s1.something), t ( SELECT something, count() AS count_a FROM table_a GROUP BY something ) s1 ALL RIGHT JOIN ( SELECT something, count() AS count_b FROM table_b GROUP BY something ) s2 -USING (something) +ON s1.something = s2.something ORDER BY count_a DESC, s1.something, s2.something; SET joined_subquery_requires_alias = 0; @@ -50,7 +50,7 @@ SELECT something, count_a, count_b, toTypeName(something) FROM ( SELECT something, count() AS count_a FROM table_a GROUP BY something ) as s1 ALL FULL JOIN ( SELECT something, count() AS count_b FROM table_b GROUP BY something ) as s2 -USING (something) +ON s1.something = s2.something ORDER BY count_a DESC, something DESC; DROP TABLE table_a; diff --git a/tests/queries/0_stateless/00858_issue_4756.reference b/tests/queries/0_stateless/00858_issue_4756.reference index d00491fd7e5..e8183f05f5d 100644 --- a/tests/queries/0_stateless/00858_issue_4756.reference +++ b/tests/queries/0_stateless/00858_issue_4756.reference @@ -1 +1,3 @@ 1 +1 +1 diff --git a/tests/queries/0_stateless/00858_issue_4756.sql b/tests/queries/0_stateless/00858_issue_4756.sql index 3da0766c4e9..9eacd5ef364 100644 --- a/tests/queries/0_stateless/00858_issue_4756.sql +++ b/tests/queries/0_stateless/00858_issue_4756.sql @@ -1,3 +1,4 @@ +set allow_experimental_analyzer = 1; set distributed_product_mode = 'local'; drop table if exists shard1; @@ -21,7 +22,7 @@ where distr1.id in from distr1 join distr2 on distr1.id = distr2.id where distr1.id > 0 -); -- { serverError 288 } +); select distinct(d0.id) from distr1 d0 where d0.id in @@ -32,15 +33,14 @@ where d0.id in where d1.id > 0 ); --- TODO ---select distinct(distr1.id) from distr1 ---where distr1.id in ---( --- select distr1.id --- from distr1 as d1 --- join distr2 as d2 on distr1.id = distr2.id --- where distr1.id > 0 ---); +select distinct(distr1.id) from distr1 +where distr1.id in +( + select distr1.id + from distr1 as d1 + join distr2 as d2 on distr1.id = distr2.id + where distr1.id > 0 +); drop table shard1; drop table shard2; diff --git a/tests/queries/0_stateless/00933_test_fix_extra_seek_on_compressed_cache.reference b/tests/queries/0_stateless/00933_test_fix_extra_seek_on_compressed_cache.reference index f1839bae259..e142c6c79fe 100644 --- a/tests/queries/0_stateless/00933_test_fix_extra_seek_on_compressed_cache.reference +++ b/tests/queries/0_stateless/00933_test_fix_extra_seek_on_compressed_cache.reference @@ -1 +1,3 @@ +99999 +99999 0 0 13 diff --git a/tests/queries/0_stateless/00933_test_fix_extra_seek_on_compressed_cache.sh b/tests/queries/0_stateless/00933_test_fix_extra_seek_on_compressed_cache.sh index 390d6a70ef1..7bf4a88e972 100755 --- a/tests/queries/0_stateless/00933_test_fix_extra_seek_on_compressed_cache.sh +++ b/tests/queries/0_stateless/00933_test_fix_extra_seek_on_compressed_cache.sh @@ -13,15 +13,24 @@ $CLICKHOUSE_CLIENT --query="CREATE TABLE small_table (a UInt64 default 0, n UInt $CLICKHOUSE_CLIENT --query="INSERT INTO small_table (n) SELECT * from system.numbers limit 100000;" $CLICKHOUSE_CLIENT --query="OPTIMIZE TABLE small_table FINAL;" -cached_query="SELECT count() FROM small_table where n > 0;" +cached_query="SELECT count() FROM small_table WHERE n > 0;" -$CLICKHOUSE_CLIENT --use_uncompressed_cache=1 --query="$cached_query" &> /dev/null - -$CLICKHOUSE_CLIENT --use_uncompressed_cache=1 --allow_prefetched_read_pool_for_remote_filesystem=0 --allow_prefetched_read_pool_for_local_filesystem=0 --query_id="test-query-uncompressed-cache" --query="$cached_query" &> /dev/null +$CLICKHOUSE_CLIENT --log_queries 1 --use_uncompressed_cache 1 --query="$cached_query" +$CLICKHOUSE_CLIENT --log_queries 1 --use_uncompressed_cache 1 --allow_prefetched_read_pool_for_remote_filesystem 0 --allow_prefetched_read_pool_for_local_filesystem 0 --query_id="test-query-uncompressed-cache" --query="$cached_query" $CLICKHOUSE_CLIENT --query="SYSTEM FLUSH LOGS" - -$CLICKHOUSE_CLIENT --query="SELECT ProfileEvents['Seek'], ProfileEvents['ReadCompressedBytes'], ProfileEvents['UncompressedCacheHits'] AS hit FROM system.query_log WHERE (query_id = 'test-query-uncompressed-cache') and current_database = currentDatabase() AND (type = 2) AND event_date >= yesterday() ORDER BY event_time DESC LIMIT 1" +$CLICKHOUSE_CLIENT --query=" + SELECT + ProfileEvents['Seek'], + ProfileEvents['ReadCompressedBytes'], + ProfileEvents['UncompressedCacheHits'] AS hit + FROM system.query_log + WHERE query_id = 'test-query-uncompressed-cache' + AND current_database = currentDatabase() + AND type = 2 + AND event_date >= yesterday() + ORDER BY event_time DESC + LIMIT 1" $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS small_table" diff --git a/tests/queries/0_stateless/00936_function_result_with_operator_in.sql b/tests/queries/0_stateless/00936_function_result_with_operator_in.sql index 9d976e5333d..0b253021f39 100644 --- a/tests/queries/0_stateless/00936_function_result_with_operator_in.sql +++ b/tests/queries/0_stateless/00936_function_result_with_operator_in.sql @@ -1,4 +1,3 @@ -SET allow_experimental_analyzer = 1; SET force_primary_key = 1; DROP TABLE IF EXISTS samples; @@ -23,8 +22,8 @@ SELECT 'a' IN splitByChar('c', 'abcdef'); SELECT 'errors:'; -- non-constant expressions in the right side of IN -SELECT count() FROM samples WHERE 1 IN range(samples.value); -- { serverError 1 } -SELECT count() FROM samples WHERE 1 IN range(rand() % 1000); -- { serverError 1 } +SELECT count() FROM samples WHERE 1 IN range(samples.value); -- { serverError 1, 47 } +SELECT count() FROM samples WHERE 1 IN range(rand() % 1000); -- { serverError 1, 36 } -- index is not used SELECT count() FROM samples WHERE value IN range(3); -- { serverError 277 } @@ -32,4 +31,4 @@ SELECT count() FROM samples WHERE value IN range(3); -- { serverError 277 } -- wrong type SELECT 123 IN splitByChar('c', 'abcdef'); -- { serverError 53 } -DROP TABLE samples; \ No newline at end of file +DROP TABLE samples; diff --git a/tests/queries/0_stateless/00965_logs_level_bugfix.reference b/tests/queries/0_stateless/00965_logs_level_bugfix.reference index 52396b3fe79..affd41b780b 100644 --- a/tests/queries/0_stateless/00965_logs_level_bugfix.reference +++ b/tests/queries/0_stateless/00965_logs_level_bugfix.reference @@ -2,7 +2,6 @@ . . - . - diff --git a/tests/queries/0_stateless/00975_values_list.sql b/tests/queries/0_stateless/00975_values_list.sql index 40c86898966..35afc99e93e 100644 --- a/tests/queries/0_stateless/00975_values_list.sql +++ b/tests/queries/0_stateless/00975_values_list.sql @@ -12,8 +12,8 @@ SELECT * FROM VALUES('n UInt64, s String, ss String', (1 + 22, '23', toString(23 SELECT * FROM VALUES('a Decimal(4, 4), b String, c String', (divide(toDecimal32(5, 3), 3), 'a', 'b')); -SELECT * FROM VALUES('x Float64', toUInt64(-1)); -- { serverError 69; } -SELECT * FROM VALUES('x Float64', NULL); -- { serverError 53; } +SELECT * FROM VALUES('x Float64', toUInt64(-1)); -- { serverError 69 } +SELECT * FROM VALUES('x Float64', NULL); -- { serverError 53 } SELECT * FROM VALUES('x Nullable(Float64)', NULL); DROP TABLE values_list; diff --git a/tests/queries/0_stateless/00988_expansion_aliases_limit.sql b/tests/queries/0_stateless/00988_expansion_aliases_limit.sql index 15c9f82da6f..3c2442b15b5 100644 --- a/tests/queries/0_stateless/00988_expansion_aliases_limit.sql +++ b/tests/queries/0_stateless/00988_expansion_aliases_limit.sql @@ -1 +1 @@ -SELECT 1 AS a, a + a AS b, b + b AS c, c + c AS d, d + d AS e, e + e AS f, f + f AS g, g + g AS h, h + h AS i, i + i AS j, j + j AS k, k + k AS l, l + l AS m, m + m AS n, n + n AS o, o + o AS p, p + p AS q, q + q AS r, r + r AS s, s + s AS t, t + t AS u, u + u AS v, v + v AS w, w + w AS x, x + x AS y, y + y AS z; -- { serverError 168 } +SELECT 1 AS a, a + a AS b, b + b AS c, c + c AS d, d + d AS e, e + e AS f, f + f AS g, g + g AS h, h + h AS i, i + i AS j, j + j AS k, k + k AS l, l + l AS m, m + m AS n, n + n AS o, o + o AS p, p + p AS q, q + q AS r, r + r AS s, s + s AS t, t + t AS u, u + u AS v, v + v AS w, w + w AS x, x + x AS y, y + y AS z; -- { serverError 36, 168 } diff --git a/tests/queries/0_stateless/01010_pm_join_all_join_bug.sql b/tests/queries/0_stateless/01010_pm_join_all_join_bug.sql index d192b9a8b64..278aa46a479 100644 --- a/tests/queries/0_stateless/01010_pm_join_all_join_bug.sql +++ b/tests/queries/0_stateless/01010_pm_join_all_join_bug.sql @@ -1,5 +1,3 @@ -SET allow_experimental_analyzer = 1; - DROP TABLE IF EXISTS ints; CREATE TABLE ints (i64 Int64, i32 Int32) ENGINE = Memory; @@ -12,6 +10,6 @@ SELECT '-'; SELECT * FROM ints l INNER JOIN ints r USING i64 ORDER BY l.i32, r.i32; SELECT '-'; -SELECT count() FROM ( SELECT [1], count(1) ) AS t1 ALL RIGHT JOIN ( SELECT number AS s FROM numbers(2) ) AS t2 USING (s); -- { serverError UNKNOWN_IDENTIFIER } +SELECT count() FROM ( SELECT [1], count(1) ) AS t1 ALL RIGHT JOIN ( SELECT number AS s FROM numbers(2) ) AS t2 USING (s); -- { serverError NOT_FOUND_COLUMN_IN_BLOCK, UNKNOWN_IDENTIFIER } DROP TABLE ints; diff --git a/tests/queries/0_stateless/01010_pmj_right_table_memory_limits.sql b/tests/queries/0_stateless/01010_pmj_right_table_memory_limits.sql index 7804ce32a5a..f9f30b44700 100644 --- a/tests/queries/0_stateless/01010_pmj_right_table_memory_limits.sql +++ b/tests/queries/0_stateless/01010_pmj_right_table_memory_limits.sql @@ -3,7 +3,10 @@ SET max_memory_usage = 32000000; SET join_on_disk_max_files_to_merge = 4; -SELECT number * 200000 as n, j FROM numbers(5) nums +SELECT n, j FROM +( + SELECT number * 200000 as n FROM numbers(5) +) nums ANY LEFT JOIN ( SELECT number * 2 AS n, number AS j FROM numbers(1000000) @@ -13,14 +16,20 @@ USING n; -- { serverError 241 } SET join_algorithm = 'partial_merge'; SET default_max_bytes_in_join = 0; -SELECT number * 200000 as n, j FROM numbers(5) nums +SELECT n, j FROM +( + SELECT number * 200000 as n FROM numbers(5) +) nums ANY LEFT JOIN ( SELECT number * 2 AS n, number AS j FROM numbers(1000000) ) js2 USING n; -- { serverError 12 } -SELECT number * 200000 as n, j FROM numbers(5) nums +SELECT n, j FROM +( + SELECT number * 200000 as n FROM numbers(5) +) nums ANY LEFT JOIN ( SELECT number * 2 AS n, number AS j FROM numbers(1000000) @@ -28,7 +37,10 @@ ANY LEFT JOIN ( USING n SETTINGS max_bytes_in_join = 30000000; -- { serverError 241 } -SELECT number * 200000 as n, j FROM numbers(5) nums +SELECT n, j FROM +( + SELECT number * 200000 as n FROM numbers(5) +) nums ANY LEFT JOIN ( SELECT number * 2 AS n, number AS j FROM numbers(1000000) @@ -39,7 +51,10 @@ SETTINGS max_bytes_in_join = 10000000; SET partial_merge_join_optimizations = 1; -SELECT number * 200000 as n, j FROM numbers(5) nums +SELECT n, j FROM +( + SELECT number * 200000 as n FROM numbers(5) +) nums LEFT JOIN ( SELECT number * 2 AS n, number AS j FROM numbers(1000000) @@ -50,7 +65,10 @@ SETTINGS max_rows_in_join = 100000; SET default_max_bytes_in_join = 10000000; -SELECT number * 200000 as n, j FROM numbers(5) nums +SELECT n, j FROM +( + SELECT number * 200000 as n FROM numbers(5) +) nums JOIN ( SELECT number * 2 AS n, number AS j FROM numbers(1000000) diff --git a/tests/queries/0_stateless/01013_repeat_function.reference b/tests/queries/0_stateless/01013_repeat_function.reference index 46bb248a99a..ea0dadd524f 100644 --- a/tests/queries/0_stateless/01013_repeat_function.reference +++ b/tests/queries/0_stateless/01013_repeat_function.reference @@ -1,7 +1,7 @@ abcabcabcabcabcabcabcabcabcabc abcabcabc -sdfggsdfgg -xywq + + abcabcabcabcabcabcabcabcabcabcabcabc sdfggsdfggsdfggsdfggsdfggsdfggsdfggsdfggsdfggsdfgg @@ -20,8 +20,8 @@ sdfggsdfggsdfggsdfggsdfggsdfggsdfggsdfggsdfggsdfgg xywqxywqxywqxywqxywqxywqxywqxywqxywqxywq plkfplkfplkfplkfplkfplkfplkfplkfplkfplkf abcabcabc -abcabc -abc + + abcabcabcabcabcabcabcabcabcabcabcabc abcabcabcabcabcabcabcabcabcabc diff --git a/tests/queries/0_stateless/01013_repeat_function.sql b/tests/queries/0_stateless/01013_repeat_function.sql index 85b0c16b4ab..b29cc032f28 100644 --- a/tests/queries/0_stateless/01013_repeat_function.sql +++ b/tests/queries/0_stateless/01013_repeat_function.sql @@ -3,20 +3,20 @@ DROP TABLE IF EXISTS defaults; CREATE TABLE defaults ( strings String, - u8 UInt8, + i8 Int8, u16 UInt16, u32 UInt32, u64 UInt64 )ENGINE = Memory(); -INSERT INTO defaults values ('abc', 3, 12, 4, 56) ('sdfgg', 2, 10, 21, 200) ('xywq', 1, 4, 9, 5) ('plkf', 0, 5, 7,77); +INSERT INTO defaults values ('abc', 3, 12, 4, 56) ('sdfgg', -2, 10, 21, 200) ('xywq', -1, 4, 9, 5) ('plkf', 0, 5, 7,77); -SELECT repeat(strings, u8) FROM defaults; +SELECT repeat(strings, i8) FROM defaults; SELECT repeat(strings, u16) FROM defaults; SELECT repeat(strings, u32) from defaults; SELECT repeat(strings, u64) FROM defaults; SELECT repeat(strings, 10) FROM defaults; -SELECT repeat('abc', u8) FROM defaults; +SELECT repeat('abc', i8) FROM defaults; SELECT repeat('abc', u16) FROM defaults; SELECT repeat('abc', u32) FROM defaults; SELECT repeat('abc', u64) FROM defaults; diff --git a/tests/queries/0_stateless/01018_Distributed__shard_num.reference b/tests/queries/0_stateless/01018_Distributed__shard_num.reference index e50ae93402b..232f12ed101 100644 --- a/tests/queries/0_stateless/01018_Distributed__shard_num.reference +++ b/tests/queries/0_stateless/01018_Distributed__shard_num.reference @@ -85,7 +85,7 @@ SELECT a._shard_num, a.key, b.host_name, b.host_address IN ('::1', '127.0.0.1'), FROM dist_1 a JOIN system.clusters b ON a._shard_num = b.shard_num -WHERE b.cluster = 'test_cluster_two_shards_localhost'; -- { serverError 403 } +WHERE b.cluster = 'test_cluster_two_shards_localhost'; -- { serverError 47, 403 } SELECT 'dist_3'; dist_3 SELECT * FROM dist_3; diff --git a/tests/queries/0_stateless/01018_Distributed__shard_num.sql b/tests/queries/0_stateless/01018_Distributed__shard_num.sql index ce522227006..7e31062348d 100644 --- a/tests/queries/0_stateless/01018_Distributed__shard_num.sql +++ b/tests/queries/0_stateless/01018_Distributed__shard_num.sql @@ -2,7 +2,6 @@ -- make the order static SET max_threads = 1; -SET allow_experimental_analyzer = 1; DROP TABLE IF EXISTS mem1; DROP TABLE IF EXISTS mem2; @@ -80,7 +79,7 @@ SELECT a._shard_num, a.key, b.host_name, b.host_address IN ('::1', '127.0.0.1'), FROM dist_1 a JOIN system.clusters b ON a._shard_num = b.shard_num -WHERE b.cluster = 'test_cluster_two_shards_localhost'; -- { serverError 403 } +WHERE b.cluster = 'test_cluster_two_shards_localhost'; -- { serverError 47, 403 } SELECT 'dist_3'; SELECT * FROM dist_3; diff --git a/tests/queries/0_stateless/01018_ambiguous_column.reference b/tests/queries/0_stateless/01018_ambiguous_column.reference index a2a1d6ea4f6..308726fa184 100644 --- a/tests/queries/0_stateless/01018_ambiguous_column.reference +++ b/tests/queries/0_stateless/01018_ambiguous_column.reference @@ -1,12 +1,15 @@ 0 0 0 0 +0 0 0 0 0 0 -┌─one.dummy─┬─A.dummy─┬─B.dummy─┐ -│ 0 │ 0 │ 0 │ -└───────────┴─────────┴─────────┘ +0 +0 +┌─system.one.dummy─┬─A.dummy─┬─B.dummy─┐ +│ 0 │ 0 │ 0 │ +└──────────────────┴─────────┴─────────┘ ┌─A.dummy─┬─one.dummy─┬─two.dummy─┐ │ 0 │ 0 │ 0 │ └─────────┴───────────┴───────────┘ diff --git a/tests/queries/0_stateless/01018_ambiguous_column.sql b/tests/queries/0_stateless/01018_ambiguous_column.sql index 54603aab810..620bdb6ba3f 100644 --- a/tests/queries/0_stateless/01018_ambiguous_column.sql +++ b/tests/queries/0_stateless/01018_ambiguous_column.sql @@ -1,4 +1,6 @@ -select * from system.one cross join system.one; -- { serverError 352 } +SET allow_experimental_analyzer = 1; + +select * from system.one cross join system.one; select * from system.one cross join system.one r; select * from system.one l cross join system.one; select * from system.one left join system.one using dummy; @@ -8,10 +10,10 @@ USE system; SELECT dummy FROM one AS A JOIN one ON A.dummy = one.dummy; SELECT dummy FROM one JOIN one AS A ON A.dummy = one.dummy; -SELECT dummy FROM one l JOIN one r ON dummy = r.dummy; -- { serverError 352 } -SELECT dummy FROM one l JOIN one r ON l.dummy = dummy; -- { serverError 352 } -SELECT dummy FROM one l JOIN one r ON one.dummy = r.dummy; -- { serverError 352 } -SELECT dummy FROM one l JOIN one r ON l.dummy = one.dummy; -- { serverError 352 } +SELECT dummy FROM one l JOIN one r ON dummy = r.dummy; +SELECT dummy FROM one l JOIN one r ON l.dummy = dummy; -- { serverError 403 } +SELECT dummy FROM one l JOIN one r ON one.dummy = r.dummy; +SELECT dummy FROM one l JOIN one r ON l.dummy = one.dummy; -- { serverError 403 } SELECT * from one JOIN one A ON one.dummy = A.dummy diff --git a/tests/queries/0_stateless/01019_alter_materialized_view_consistent.sh b/tests/queries/0_stateless/01019_alter_materialized_view_consistent.sh index e90085f4e8e..3a2eac1f38f 100755 --- a/tests/queries/0_stateless/01019_alter_materialized_view_consistent.sh +++ b/tests/queries/0_stateless/01019_alter_materialized_view_consistent.sh @@ -50,12 +50,20 @@ function insert_thread() { function alter_thread() { trap 'exit' INT - ALTER[0]="ALTER TABLE mv MODIFY QUERY SELECT v == 1 as test, v as case FROM src_a;" - ALTER[1]="ALTER TABLE mv MODIFY QUERY SELECT v == 2 as test, v as case FROM src_b;" + # Generate random ALTERs, but make sure that at least one of them is for each source table. + for i in {0..5}; do + ALTER[$i]="ALTER TABLE mv MODIFY QUERY SELECT v == 1 as test, v as case FROM src_a;" + done + # Insert 3 ALTERs to src_b, one in the first half of the array and two in arbitrary positions. + ALTER[$RANDOM % 3]="ALTER TABLE mv MODIFY QUERY SELECT v == 2 as test, v as case FROM src_b;" + ALTER[$RANDOM % 6]="ALTER TABLE mv MODIFY QUERY SELECT v == 2 as test, v as case FROM src_b;" + ALTER[$RANDOM % 6]="ALTER TABLE mv MODIFY QUERY SELECT v == 2 as test, v as case FROM src_b;" + i=0 while true; do - $CLICKHOUSE_CLIENT --allow_experimental_alter_materialized_view_structure=1 \ - -q "${ALTER[$RANDOM % 2]}" + $CLICKHOUSE_CLIENT --allow_experimental_alter_materialized_view_structure=1 -q "${ALTER[$i % 6]}" + ((i=i+1)) + sleep "0.0$RANDOM" is_done=$($CLICKHOUSE_CLIENT -q "SELECT countIf(case = 1) > 0 AND countIf(case = 2) > 0 FROM mv;") diff --git a/tests/queries/0_stateless/01040_dictionary_invalidate_query_switchover_long.reference b/tests/queries/0_stateless/01040_dictionary_invalidate_query_switchover_long.reference index c89fe48d9f9..8d40aebacf2 100644 --- a/tests/queries/0_stateless/01040_dictionary_invalidate_query_switchover_long.reference +++ b/tests/queries/0_stateless/01040_dictionary_invalidate_query_switchover_long.reference @@ -1,5 +1,5 @@ 122 -Table dictdb_01041_01040.dict_invalidate doesn\'t exist +1 133 diff --git a/tests/queries/0_stateless/01040_dictionary_invalidate_query_switchover_long.sh b/tests/queries/0_stateless/01040_dictionary_invalidate_query_switchover_long.sh index 7249d5e1a82..6856f952a47 100755 --- a/tests/queries/0_stateless/01040_dictionary_invalidate_query_switchover_long.sh +++ b/tests/queries/0_stateless/01040_dictionary_invalidate_query_switchover_long.sh @@ -53,7 +53,7 @@ function check_exception_detected() export -f check_exception_detected; timeout 30 bash -c check_exception_detected 2> /dev/null -$CLICKHOUSE_CLIENT --query "SELECT last_exception FROM system.dictionaries WHERE database = 'dictdb_01041_01040' AND name = 'invalidate'" 2>&1 | grep -Eo "Table dictdb_01041_01040.dict_invalidate .* exist" +$CLICKHOUSE_CLIENT --query "SELECT last_exception FROM system.dictionaries WHERE database = 'dictdb_01041_01040' AND name = 'invalidate'" 2>&1 | grep -Eo "dictdb_01041_01040.dict_invalidate.*UNKNOWN_TABLE" | wc -l $CLICKHOUSE_CLIENT --query " CREATE TABLE dictdb_01041_01040.dict_invalidate diff --git a/tests/queries/0_stateless/01047_window_view_parser_inner_table.sql b/tests/queries/0_stateless/01047_window_view_parser_inner_table.sql index 2d9911287a3..bf1ac254783 100644 --- a/tests/queries/0_stateless/01047_window_view_parser_inner_table.sql +++ b/tests/queries/0_stateless/01047_window_view_parser_inner_table.sql @@ -1,5 +1,6 @@ -- Tags: no-parallel +SET allow_experimental_analyzer = 0; SET allow_experimental_window_view = 1; DROP DATABASE IF EXISTS test_01047; set allow_deprecated_database_ordinary=1; diff --git a/tests/queries/0_stateless/01048_window_view_parser.sql b/tests/queries/0_stateless/01048_window_view_parser.sql index 4c329f99f6e..f87d9aa023e 100644 --- a/tests/queries/0_stateless/01048_window_view_parser.sql +++ b/tests/queries/0_stateless/01048_window_view_parser.sql @@ -1,5 +1,6 @@ -- Tags: no-parallel +SET allow_experimental_analyzer = 0; SET allow_experimental_window_view = 1; DROP DATABASE IF EXISTS test_01048; set allow_deprecated_database_ordinary=1; diff --git a/tests/queries/0_stateless/01050_window_view_parser_tumble.sql b/tests/queries/0_stateless/01050_window_view_parser_tumble.sql index d9604bb2b52..f49fbc251fd 100644 --- a/tests/queries/0_stateless/01050_window_view_parser_tumble.sql +++ b/tests/queries/0_stateless/01050_window_view_parser_tumble.sql @@ -1,3 +1,4 @@ +SET allow_experimental_analyzer = 0; SET allow_experimental_window_view = 1; DROP TABLE IF EXISTS mt; diff --git a/tests/queries/0_stateless/01051_window_view_parser_hop.sql b/tests/queries/0_stateless/01051_window_view_parser_hop.sql index 472dc66f1a2..45877cf0647 100644 --- a/tests/queries/0_stateless/01051_window_view_parser_hop.sql +++ b/tests/queries/0_stateless/01051_window_view_parser_hop.sql @@ -1,3 +1,4 @@ +SET allow_experimental_analyzer = 0; SET allow_experimental_window_view = 1; DROP TABLE IF EXISTS mt; diff --git a/tests/queries/0_stateless/01052_window_view_proc_tumble_to_now.sh b/tests/queries/0_stateless/01052_window_view_proc_tumble_to_now.sh index 9fdc66191d7..e75b7d9570b 100755 --- a/tests/queries/0_stateless/01052_window_view_proc_tumble_to_now.sh +++ b/tests/queries/0_stateless/01052_window_view_proc_tumble_to_now.sh @@ -4,7 +4,11 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh -$CLICKHOUSE_CLIENT --multiquery <&1| grep -Fa "Exception: " | grep -Fv "REPLICA_ALREADY_EXISTS" | grep -Fiv "Will not try to start it up" | \ grep -Fv "Coordination::Exception" | grep -Fv "already contains some data and it does not look like Replicated database path" sleep 0.$RANDOM diff --git a/tests/queries/0_stateless/01165_lost_part_empty_partition.sql b/tests/queries/0_stateless/01165_lost_part_empty_partition.sql index acda7d68391..dc41b15118f 100644 --- a/tests/queries/0_stateless/01165_lost_part_empty_partition.sql +++ b/tests/queries/0_stateless/01165_lost_part_empty_partition.sql @@ -1,13 +1,11 @@ -- Tags: zookeeper -SET allow_experimental_analyzer = 1; - create table rmt1 (d DateTime, n int) engine=ReplicatedMergeTree('/test/01165/{database}/rmt', '1') order by n partition by toYYYYMMDD(d); create table rmt2 (d DateTime, n int) engine=ReplicatedMergeTree('/test/01165/{database}/rmt', '2') order by n partition by toYYYYMMDD(d); system stop replicated sends rmt1; insert into rmt1 values (now(), arrayJoin([1, 2])); -- { clientError 36 } -insert into rmt1(n) select * from system.numbers limit arrayJoin([1, 2]); -- { serverError 440 } +insert into rmt1(n) select * from system.numbers limit arrayJoin([1, 2]); -- { serverError 36, 440 } insert into rmt1 values (now(), rand()); drop table rmt1; diff --git a/tests/queries/0_stateless/01169_alter_partition_isolation_stress.sh b/tests/queries/0_stateless/01169_alter_partition_isolation_stress.sh index a385fc81fe4..508ad05224c 100755 --- a/tests/queries/0_stateless/01169_alter_partition_isolation_stress.sh +++ b/tests/queries/0_stateless/01169_alter_partition_isolation_stress.sh @@ -2,6 +2,7 @@ # Tags: long, no-replicated-database, no-ordinary-database # shellcheck disable=SC2015 +# shellcheck disable=SC2119 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh diff --git a/tests/queries/0_stateless/01171_mv_select_insert_isolation_long.sh b/tests/queries/0_stateless/01171_mv_select_insert_isolation_long.sh index 12b654f4215..199c2b5389f 100755 --- a/tests/queries/0_stateless/01171_mv_select_insert_isolation_long.sh +++ b/tests/queries/0_stateless/01171_mv_select_insert_isolation_long.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash # Tags: long, no-parallel, no-ordinary-database # Test is too heavy, avoid parallel run in Flaky Check +# shellcheck disable=SC2119 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh diff --git a/tests/queries/0_stateless/01174_select_insert_isolation.sh b/tests/queries/0_stateless/01174_select_insert_isolation.sh index dc5c1d7a722..29ccfbb1ccb 100755 --- a/tests/queries/0_stateless/01174_select_insert_isolation.sh +++ b/tests/queries/0_stateless/01174_select_insert_isolation.sh @@ -2,6 +2,7 @@ # Tags: long, no-ordinary-database # shellcheck disable=SC2015 +# shellcheck disable=SC2119 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh diff --git a/tests/queries/0_stateless/01195_formats_diagnostic_info.sh b/tests/queries/0_stateless/01195_formats_diagnostic_info.sh index e75780a4520..b146d65fc58 100755 --- a/tests/queries/0_stateless/01195_formats_diagnostic_info.sh +++ b/tests/queries/0_stateless/01195_formats_diagnostic_info.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# Tags: no-fasttest # shellcheck disable=SC2206 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) diff --git a/tests/queries/0_stateless/01201_read_single_thread_in_order.sql b/tests/queries/0_stateless/01201_read_single_thread_in_order.sql index 33ccbbbe84d..24ed935a125 100644 --- a/tests/queries/0_stateless/01201_read_single_thread_in_order.sql +++ b/tests/queries/0_stateless/01201_read_single_thread_in_order.sql @@ -6,7 +6,7 @@ CREATE TABLE t ) ENGINE = MergeTree ORDER BY number -SETTINGS index_granularity = 128, index_granularity_bytes = '10Mi'; +SETTINGS index_granularity = 128, ratio_of_defaults_for_sparse_serialization = 1.0, index_granularity_bytes = '10Mi'; SET min_insert_block_size_rows = 0, min_insert_block_size_bytes = 0; INSERT INTO t SELECT number FROM numbers(10000000); diff --git a/tests/queries/0_stateless/01211_optimize_skip_unused_shards_type_mismatch.sql b/tests/queries/0_stateless/01211_optimize_skip_unused_shards_type_mismatch.sql index 65adaf3ad71..de41132df62 100644 --- a/tests/queries/0_stateless/01211_optimize_skip_unused_shards_type_mismatch.sql +++ b/tests/queries/0_stateless/01211_optimize_skip_unused_shards_type_mismatch.sql @@ -9,7 +9,7 @@ create table data_02000 (key Int) Engine=Null(); create table dist_02000 as data_02000 Engine=Distributed(test_cluster_two_shards, currentDatabase(), data_02000, key); select * from data_02000 where key = 0xdeadbeafdeadbeaf; -select * from dist_02000 where key = 0xdeadbeafdeadbeaf settings force_optimize_skip_unused_shards=2; -- { serverError 507; } +select * from dist_02000 where key = 0xdeadbeafdeadbeaf settings force_optimize_skip_unused_shards=2; -- { serverError 507 } select * from dist_02000 where key = 0xdeadbeafdeadbeaf; drop table data_02000; diff --git a/tests/queries/0_stateless/01225_drop_dictionary_as_table.sql b/tests/queries/0_stateless/01225_drop_dictionary_as_table.sql index 513ecbd4ed4..be2f7b2a9bf 100644 --- a/tests/queries/0_stateless/01225_drop_dictionary_as_table.sql +++ b/tests/queries/0_stateless/01225_drop_dictionary_as_table.sql @@ -16,7 +16,7 @@ LAYOUT(FLAT()); SYSTEM RELOAD DICTIONARY dict_db_01225.dict; -DROP TABLE dict_db_01225.dict; -- { serverError 520; } +DROP TABLE dict_db_01225.dict; -- { serverError 520 } DROP DICTIONARY dict_db_01225.dict; DROP DATABASE dict_db_01225; diff --git a/tests/queries/0_stateless/01225_show_create_table_from_dictionary.sql b/tests/queries/0_stateless/01225_show_create_table_from_dictionary.sql index 09cde642ed2..bc733a0c546 100644 --- a/tests/queries/0_stateless/01225_show_create_table_from_dictionary.sql +++ b/tests/queries/0_stateless/01225_show_create_table_from_dictionary.sql @@ -18,7 +18,7 @@ LIFETIME(MIN 0 MAX 0) LAYOUT(FLAT()); SHOW CREATE TABLE dict_db_01225_dictionary.`dict_db_01225.dict` FORMAT TSVRaw; -SHOW CREATE TABLE dict_db_01225_dictionary.`dict_db_01225.no_such_dict`; -- { serverError 487; } +SHOW CREATE TABLE dict_db_01225_dictionary.`dict_db_01225.no_such_dict`; -- { serverError 487 } DROP DATABASE dict_db_01225; DROP DATABASE dict_db_01225_dictionary; diff --git a/tests/queries/0_stateless/01231_log_queries_min_type.sql b/tests/queries/0_stateless/01231_log_queries_min_type.sql index c2470bb9a56..0ed5e3e605c 100644 --- a/tests/queries/0_stateless/01231_log_queries_min_type.sql +++ b/tests/queries/0_stateless/01231_log_queries_min_type.sql @@ -15,7 +15,7 @@ select count() from system.query_log where current_database = currentDatabase() set max_rows_to_read='100K'; set log_queries_min_type='EXCEPTION_WHILE_PROCESSING'; -select '01231_log_queries_min_type/EXCEPTION_WHILE_PROCESSING', max(number) from system.numbers limit 1e6; -- { serverError 158; } +select '01231_log_queries_min_type/EXCEPTION_WHILE_PROCESSING', max(number) from system.numbers limit 1e6; -- { serverError 158 } set max_rows_to_read=0; system flush logs; select count() from system.query_log where current_database = currentDatabase() @@ -23,7 +23,7 @@ select count() from system.query_log where current_database = currentDatabase() and event_date >= yesterday() and type = 'ExceptionWhileProcessing'; set max_rows_to_read='100K'; -select '01231_log_queries_min_type w/ Settings/EXCEPTION_WHILE_PROCESSING', max(number) from system.numbers limit 1e6; -- { serverError 158; } +select '01231_log_queries_min_type w/ Settings/EXCEPTION_WHILE_PROCESSING', max(number) from system.numbers limit 1e6; -- { serverError 158 } system flush logs; set max_rows_to_read=0; select count() from system.query_log where diff --git a/tests/queries/0_stateless/01246_buffer_flush.sql b/tests/queries/0_stateless/01246_buffer_flush.sql index 47891a7f00e..ac507d94b69 100644 --- a/tests/queries/0_stateless/01246_buffer_flush.sql +++ b/tests/queries/0_stateless/01246_buffer_flush.sql @@ -1,3 +1,5 @@ +-- Tags: no-fasttest + drop table if exists data_01256; drop table if exists buffer_01256; diff --git a/tests/queries/0_stateless/01271_show_privileges.reference b/tests/queries/0_stateless/01271_show_privileges.reference index c061eb95a65..abebc35d072 100644 --- a/tests/queries/0_stateless/01271_show_privileges.reference +++ b/tests/queries/0_stateless/01271_show_privileges.reference @@ -50,7 +50,8 @@ CREATE DATABASE [] DATABASE CREATE CREATE TABLE [] TABLE CREATE CREATE VIEW [] VIEW CREATE CREATE DICTIONARY [] DICTIONARY CREATE -CREATE TEMPORARY TABLE [] GLOBAL CREATE +CREATE TEMPORARY TABLE [] GLOBAL CREATE ARBITRARY TEMPORARY TABLE +CREATE ARBITRARY TEMPORARY TABLE [] GLOBAL CREATE CREATE FUNCTION [] GLOBAL CREATE CREATE NAMED COLLECTION [] GLOBAL CREATE CREATE [] \N ALL diff --git a/tests/queries/0_stateless/01278_min_insert_block_size_rows_for_materialized_views.sh b/tests/queries/0_stateless/01278_min_insert_block_size_rows_for_materialized_views.sh index 0e258bbbb09..08cc97c84bf 100755 --- a/tests/queries/0_stateless/01278_min_insert_block_size_rows_for_materialized_views.sh +++ b/tests/queries/0_stateless/01278_min_insert_block_size_rows_for_materialized_views.sh @@ -76,7 +76,7 @@ insert into data_01278 select reinterpretAsString(number), // s6 reinterpretAsString(number), // s7 reinterpretAsString(number) // s8 -from numbers(100000); -- { serverError 241; }" > /dev/null 2>&1 +from numbers(100000); -- { serverError 241 }" > /dev/null 2>&1 local ret_code=$? if [[ $ret_code -eq 0 ]]; then diff --git a/tests/queries/0_stateless/01284_port.sql.j2 b/tests/queries/0_stateless/01284_port.sql.j2 index 6f78b3b8e3b..50e096c6deb 100644 --- a/tests/queries/0_stateless/01284_port.sql.j2 +++ b/tests/queries/0_stateless/01284_port.sql.j2 @@ -19,9 +19,9 @@ select port{{ suffix }}('http://127.0.0.1/', toUInt16(80)); select port{{ suffix }}('http://foobar.com/', toUInt16(80)); -- unsupported -/* ILLEGAL_TYPE_OF_ARGUMENT */ select port(toFixedString('', 1)); -- { serverError 43; } -/* ILLEGAL_TYPE_OF_ARGUMENT */ select port{{ suffix }}('', 1); -- { serverError 43; } -/* NUMBER_OF_ARGUMENTS_DOESNT_MATCH */ select port{{ suffix }}('', 1, 1); -- { serverError 42; } +/* ILLEGAL_TYPE_OF_ARGUMENT */ select port(toFixedString('', 1)); -- { serverError 43 } +/* ILLEGAL_TYPE_OF_ARGUMENT */ select port{{ suffix }}('', 1); -- { serverError 43 } +/* NUMBER_OF_ARGUMENTS_DOESNT_MATCH */ select port{{ suffix }}('', 1, 1); -- { serverError 42 } -- -- Known limitations of domain() (getURLHost()) diff --git a/tests/queries/0_stateless/01291_aggregation_in_order.reference b/tests/queries/0_stateless/01291_aggregation_in_order.reference index c072a8aed3e..cf058b9f2f5 100644 --- a/tests/queries/0_stateless/01291_aggregation_in_order.reference +++ b/tests/queries/0_stateless/01291_aggregation_in_order.reference @@ -22,8 +22,8 @@ 2 4 109 2 1 619 1 2 537 2 -1 619 1 -2 537 2 +-2 537 2 +-1 619 1 2019-05-05 00:00:00 -45363190 2019-05-05 00:00:00 -1249512288 2019-05-05 00:00:00 345522721 diff --git a/tests/queries/0_stateless/01291_aggregation_in_order.sql b/tests/queries/0_stateless/01291_aggregation_in_order.sql index c4357811520..e93eadc3329 100644 --- a/tests/queries/0_stateless/01291_aggregation_in_order.sql +++ b/tests/queries/0_stateless/01291_aggregation_in_order.sql @@ -14,7 +14,7 @@ SELECT a FROM pk_order GROUP BY a ORDER BY a; SELECT a, b, sum(c), avg(d) FROM pk_order GROUP BY a, b ORDER BY a, b; SELECT a, sum(c), avg(d) FROM pk_order GROUP BY a ORDER BY a; -SELECT a, sum(c), avg(d) FROM pk_order GROUP BY -a ORDER BY a; +SELECT -a, sum(c), avg(d) FROM pk_order GROUP BY -a ORDER BY -a; DROP TABLE IF EXISTS pk_order; @@ -27,7 +27,7 @@ INSERT INTO pk_order set max_block_size = 1; SELECT d, max(b) FROM pk_order GROUP BY d, a ORDER BY d, a LIMIT 5; -SELECT d, avg(a) FROM pk_order GROUP BY toString(d) ORDER BY toString(d) LIMIT 5; +SELECT toString(d), avg(a) FROM pk_order GROUP BY toString(d) ORDER BY toString(d) LIMIT 5; SELECT toStartOfHour(d) as d1, min(a), max(b) FROM pk_order GROUP BY d1 ORDER BY d1 LIMIT 5; DROP TABLE pk_order; diff --git a/tests/queries/0_stateless/01293_optimize_final_force.sh b/tests/queries/0_stateless/01293_optimize_final_force.sh index 60d45f87385..994d5952dbc 100755 --- a/tests/queries/0_stateless/01293_optimize_final_force.sh +++ b/tests/queries/0_stateless/01293_optimize_final_force.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# Tags: no-fasttest CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh diff --git a/tests/queries/0_stateless/01300_client_save_history_when_terminated_long.expect b/tests/queries/0_stateless/01300_client_save_history_when_terminated_long.expect index a593075bb9a..c897d7e9772 100755 --- a/tests/queries/0_stateless/01300_client_save_history_when_terminated_long.expect +++ b/tests/queries/0_stateless/01300_client_save_history_when_terminated_long.expect @@ -28,7 +28,7 @@ exec kill -9 [exp_pid] close # Run client one more time and press "up" to see the last recorded query -spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --history_file=$history_file" +spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion --history_file=$history_file" expect ":) " send -- "\[A" expect "for the history" diff --git a/tests/queries/0_stateless/01308_orc_output_format_arrays.reference b/tests/queries/0_stateless/01308_orc_output_format_arrays.reference index 1f9646ac112..7feea7cec35 100644 Binary files a/tests/queries/0_stateless/01308_orc_output_format_arrays.reference and b/tests/queries/0_stateless/01308_orc_output_format_arrays.reference differ diff --git a/tests/queries/0_stateless/01308_orc_output_format_arrays.sh b/tests/queries/0_stateless/01308_orc_output_format_arrays.sh index 1d9aea353b6..498854874cf 100755 --- a/tests/queries/0_stateless/01308_orc_output_format_arrays.sh +++ b/tests/queries/0_stateless/01308_orc_output_format_arrays.sh @@ -11,7 +11,7 @@ $CLICKHOUSE_CLIENT --query="CREATE TABLE orc (array1 Array(Int32), array2 Array( $CLICKHOUSE_CLIENT --query="INSERT INTO orc VALUES ([1,2,3,4,5], [[1,2], [3,4], [5]]), ([42], [[42, 42], [42]])"; -$CLICKHOUSE_CLIENT --query="SELECT * FROM orc FORMAT ORC"; +$CLICKHOUSE_CLIENT --query="SELECT * FROM orc FORMAT ORC SETTINGS output_format_orc_compression_method='none'" | md5sum; $CLICKHOUSE_CLIENT --query="DROP TABLE orc"; diff --git a/tests/queries/0_stateless/01375_GROUP_BY_injective_elimination_dictGet_BAD_ARGUMENTS.sql b/tests/queries/0_stateless/01375_GROUP_BY_injective_elimination_dictGet_BAD_ARGUMENTS.sql index 88a2b25c2db..8ff9cd2b9f2 100644 --- a/tests/queries/0_stateless/01375_GROUP_BY_injective_elimination_dictGet_BAD_ARGUMENTS.sql +++ b/tests/queries/0_stateless/01375_GROUP_BY_injective_elimination_dictGet_BAD_ARGUMENTS.sql @@ -1 +1 @@ -SELECT dictGetString(concat('default', '.countryId'), 'country', toUInt64(number)) AS country FROM numbers(2) GROUP BY country; -- { serverError 36; } +SELECT dictGetString(concat('default', '.countryId'), 'country', toUInt64(number)) AS country FROM numbers(2) GROUP BY country; -- { serverError 36 } diff --git a/tests/queries/0_stateless/01376_GROUP_BY_injective_elimination_dictGet.sql b/tests/queries/0_stateless/01376_GROUP_BY_injective_elimination_dictGet.sql index 258d96829a5..29ffcb46fbf 100644 --- a/tests/queries/0_stateless/01376_GROUP_BY_injective_elimination_dictGet.sql +++ b/tests/queries/0_stateless/01376_GROUP_BY_injective_elimination_dictGet.sql @@ -1,7 +1,7 @@ -- Tags: no-parallel -- https://github.com/ClickHouse/ClickHouse/issues/11469 -SELECT dictGet('default.countryId', 'country', toUInt64(number)) AS country FROM numbers(2) GROUP BY country; -- { serverError 36; } +SELECT dictGet('default.countryId', 'country', toUInt64(number)) AS country FROM numbers(2) GROUP BY country; -- { serverError 36 } -- with real dictionary diff --git a/tests/queries/0_stateless/01380_coded_delta_exception_code.sql b/tests/queries/0_stateless/01380_coded_delta_exception_code.sql index 587fac958cd..f4b88a93904 100644 --- a/tests/queries/0_stateless/01380_coded_delta_exception_code.sql +++ b/tests/queries/0_stateless/01380_coded_delta_exception_code.sql @@ -2,5 +2,5 @@ CREATE TABLE delta_codec_synthetic (`id` Decimal(38, 10) CODEC(Delta, ZSTD(22))) CREATE TABLE delta_codec_synthetic (`id` Decimal(38, 10) CODEC(DoubleDelta, ZSTD(22))) ENGINE = MergeTree() ORDER BY tuple(); -- { serverError 36 } CREATE TABLE delta_codec_synthetic (`id` Decimal(38, 10) CODEC(Gorilla, ZSTD(22))) ENGINE = MergeTree() ORDER BY tuple(); -- { serverError 36 } -CREATE TABLE delta_codec_synthetic (`id` UInt64 CODEC(DoubleDelta(3), ZSTD(22))) ENGINE = MergeTree() ORDER BY tuple(); -- { serverError 36 } -CREATE TABLE delta_codec_synthetic (`id` UInt64 CODEC(Gorilla('hello, world'), ZSTD(22))) ENGINE = MergeTree() ORDER BY tuple(); -- { serverError 36 } +CREATE TABLE delta_codec_synthetic (`id` UInt64 CODEC(DoubleDelta(3), ZSTD(22))) ENGINE = MergeTree() ORDER BY tuple(); -- { serverError ILLEGAL_CODEC_PARAMETER } +CREATE TABLE delta_codec_synthetic (`id` UInt64 CODEC(Gorilla('hello, world'), ZSTD(22))) ENGINE = MergeTree() ORDER BY tuple(); -- { serverError ILLEGAL_CODEC_PARAMETER } diff --git a/tests/queries/0_stateless/01395_limit_more_cases.sh b/tests/queries/0_stateless/01395_limit_more_cases.sh index 32c854e53fb..177147d2142 100755 --- a/tests/queries/0_stateless/01395_limit_more_cases.sh +++ b/tests/queries/0_stateless/01395_limit_more_cases.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# Tags: no-fasttest CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh diff --git a/tests/queries/0_stateless/01402_cast_nullable_string_to_enum.sql b/tests/queries/0_stateless/01402_cast_nullable_string_to_enum.sql index 3b53e593095..b8b5370515a 100644 --- a/tests/queries/0_stateless/01402_cast_nullable_string_to_enum.sql +++ b/tests/queries/0_stateless/01402_cast_nullable_string_to_enum.sql @@ -5,9 +5,9 @@ SELECT CAST(CAST(NULL AS Nullable(String)) AS Nullable(Enum8('Hello' = 1))); SELECT CAST(CAST(NULL AS Nullable(FixedString(1))) AS Nullable(Enum8('Hello' = 1))); -- empty string still not acceptable -SELECT CAST(CAST('' AS Nullable(String)) AS Nullable(Enum8('Hello' = 1))); -- { serverError 36; } -SELECT CAST(CAST('' AS Nullable(FixedString(1))) AS Nullable(Enum8('Hello' = 1))); -- { serverError 36; } +SELECT CAST(CAST('' AS Nullable(String)) AS Nullable(Enum8('Hello' = 1))); -- { serverError 36 } +SELECT CAST(CAST('' AS Nullable(FixedString(1))) AS Nullable(Enum8('Hello' = 1))); -- { serverError 36 } -- non-Nullable Enum() still not acceptable -SELECT CAST(CAST(NULL AS Nullable(String)) AS Enum8('Hello' = 1)); -- { serverError 349; } -SELECT CAST(CAST(NULL AS Nullable(FixedString(1))) AS Enum8('Hello' = 1)); -- { serverError 349; } +SELECT CAST(CAST(NULL AS Nullable(String)) AS Enum8('Hello' = 1)); -- { serverError 349 } +SELECT CAST(CAST(NULL AS Nullable(FixedString(1))) AS Enum8('Hello' = 1)); -- { serverError 349 } diff --git a/tests/queries/0_stateless/01404_roundUpToPowerOfTwoOrZero_safety.sql b/tests/queries/0_stateless/01404_roundUpToPowerOfTwoOrZero_safety.sql index 4ee6e1fa5e4..d61a35c9999 100644 --- a/tests/queries/0_stateless/01404_roundUpToPowerOfTwoOrZero_safety.sql +++ b/tests/queries/0_stateless/01404_roundUpToPowerOfTwoOrZero_safety.sql @@ -1,4 +1,4 @@ -- repeat() with this length and this number of rows will allocation huge enough region (MSB set), -- which will cause roundUpToPowerOfTwoOrZero() returns 0 for such allocation (before the fix), -- and later repeat() will try to use this memory and will got SIGSEGV. -SELECT repeat('0.0001048576', number * (number * (number * 255))) FROM numbers(65535); -- { serverError 131; } +SELECT repeat('0.0001048576', number * (number * (number * 255))) FROM numbers(65535); -- { serverError 131 } diff --git a/tests/queries/0_stateless/01407_lambda_arrayJoin.sql b/tests/queries/0_stateless/01407_lambda_arrayJoin.sql index 363b1d92dbb..e1b8c1d5a76 100644 --- a/tests/queries/0_stateless/01407_lambda_arrayJoin.sql +++ b/tests/queries/0_stateless/01407_lambda_arrayJoin.sql @@ -1,5 +1,5 @@ SELECT arrayFilter((a) -> ((a, arrayJoin([])) IN (Null, [Null])), []); SELECT arrayFilter((a) -> ((a, arrayJoin([[]])) IN (Null, [Null])), []); -SELECT * FROM system.one ARRAY JOIN arrayFilter((a) -> ((a, arrayJoin([])) IN (NULL)), []) AS arr_x; -- { serverError 43; } +SELECT * FROM system.one ARRAY JOIN arrayFilter((a) -> ((a, arrayJoin([])) IN (NULL)), []) AS arr_x; -- { serverError 43 } SELECT * FROM numbers(1) LEFT ARRAY JOIN arrayFilter((x_0, x_1) -> (arrayJoin([]) IN (NULL)), [], []) AS arr_x; diff --git a/tests/queries/0_stateless/01408_range_overflow.sql b/tests/queries/0_stateless/01408_range_overflow.sql index 2107e8c3f36..d26507f8358 100644 --- a/tests/queries/0_stateless/01408_range_overflow.sql +++ b/tests/queries/0_stateless/01408_range_overflow.sql @@ -1,7 +1,7 @@ -- executeGeneric() SELECT range(1025, 1048576 + 9223372036854775807, 9223372036854775807); SELECT range(1025, 1048576 + (9223372036854775807 AS i), i); -SELECT range(1025, 18446744073709551615, 1); -- { serverError 69; } +SELECT range(1025, 18446744073709551615, 1); -- { serverError 69 } -- executeConstStep() SELECT range(number, 1048576 + 9223372036854775807, 9223372036854775807) FROM system.numbers LIMIT 1 OFFSET 1025; diff --git a/tests/queries/0_stateless/01429_join_on_error_messages.sql b/tests/queries/0_stateless/01429_join_on_error_messages.sql index 9b8688c8415..b22d5259136 100644 --- a/tests/queries/0_stateless/01429_join_on_error_messages.sql +++ b/tests/queries/0_stateless/01429_join_on_error_messages.sql @@ -1,10 +1,8 @@ -SET allow_experimental_analyzer = 1; - SELECT 1 FROM (select 1 a) A JOIN (select 1 b) B ON (arrayJoin([1]) = B.b); -- { serverError 403 } SELECT 1 FROM (select 1 a) A JOIN (select 1 b) B ON (A.a = arrayJoin([1])); -- { serverError 403 } -SELECT 1 FROM (select 1 a) A JOIN (select 1 b) B ON equals(a); -- { serverError 42 } -SELECT 1 FROM (select 1 a) A JOIN (select 1 b) B ON less(a); -- { serverError 42 } +SELECT 1 FROM (select 1 a) A JOIN (select 1 b) B ON equals(a); -- { serverError 42, 62 } +SELECT 1 FROM (select 1 a) A JOIN (select 1 b) B ON less(a); -- { serverError 42, 62 } SELECT 1 FROM (select 1 a) A JOIN (select 1 b) B ON a = b AND a > b; -- { serverError 403 } SELECT 1 FROM (select 1 a) A JOIN (select 1 b) B ON a = b AND a < b; -- { serverError 403 } @@ -15,7 +13,7 @@ SET join_algorithm = 'partial_merge'; SELECT 1 FROM (select 1 a) A JOIN (select 1 b, 1 c) B ON a = b OR a = c; -- { serverError 48 } -- works for a = b OR a = b because of equivalent disjunct optimization -SET join_algorithm = 'auto'; +SET join_algorithm = 'grace_hash'; SELECT 1 FROM (select 1 a) A JOIN (select 1 b, 1 c) B ON a = b OR a = c; -- { serverError 48 } -- works for a = b OR a = b because of equivalent disjunct optimization diff --git a/tests/queries/0_stateless/01455_opentelemetry_distributed.reference b/tests/queries/0_stateless/01455_opentelemetry_distributed.reference index d48b3738bc2..9343593e1ba 100644 --- a/tests/queries/0_stateless/01455_opentelemetry_distributed.reference +++ b/tests/queries/0_stateless/01455_opentelemetry_distributed.reference @@ -1,10 +1,10 @@ ===http=== -{"query":"select 1 from remote('127.0.0.2', system, one) format Null\n","status":"QueryFinish","tracestate":"some custom state","sorted_by_start_time":1} +{"query":"select 1 from remote('127.0.0.2', system, one) settings allow_experimental_analyzer = 1 format Null\n","status":"QueryFinish","tracestate":"some custom state","sorted_by_start_time":1} {"query":"DESC TABLE system.one","status":"QueryFinish","tracestate":"some custom state","sorted_by_start_time":1} -{"query":"SELECT 1 FROM `system`.`one`","status":"QueryFinish","tracestate":"some custom state","sorted_by_start_time":1} +{"query":"SELECT 1 AS `1` FROM `system`.`one`","status":"QueryFinish","tracestate":"some custom state","sorted_by_start_time":1} {"query":"DESC TABLE system.one","query_status":"QueryFinish","tracestate":"some custom state","sorted_by_finish_time":1} -{"query":"SELECT 1 FROM `system`.`one`","query_status":"QueryFinish","tracestate":"some custom state","sorted_by_finish_time":1} -{"query":"select 1 from remote('127.0.0.2', system, one) format Null\n","query_status":"QueryFinish","tracestate":"some custom state","sorted_by_finish_time":1} +{"query":"SELECT 1 AS `1` FROM `system`.`one`","query_status":"QueryFinish","tracestate":"some custom state","sorted_by_finish_time":1} +{"query":"select 1 from remote('127.0.0.2', system, one) settings allow_experimental_analyzer = 1 format Null\n","query_status":"QueryFinish","tracestate":"some custom state","sorted_by_finish_time":1} {"total spans":"3","unique spans":"3","unique non-zero parent spans":"3"} {"initial query spans with proper parent":"1"} {"unique non-empty tracestate values":"1"} @@ -12,10 +12,12 @@ {"query":"select * from url('http:\/\/127.0.0.2:8123\/?query=select%201%20format%20Null', CSV, 'a int')","status":"QueryFinish","tracestate":"another custom state","sorted_by_start_time":1} {"query":"select 1 format Null\n","status":"QueryFinish","tracestate":"another custom state","sorted_by_start_time":1} {"query":"select 1 format Null\n","status":"QueryFinish","tracestate":"another custom state","sorted_by_start_time":1} +{"query":"select 1 format Null\n","status":"QueryFinish","tracestate":"another custom state","sorted_by_start_time":1} +{"query":"select 1 format Null\n","query_status":"QueryFinish","tracestate":"another custom state","sorted_by_finish_time":1} {"query":"select 1 format Null\n","query_status":"QueryFinish","tracestate":"another custom state","sorted_by_finish_time":1} {"query":"select 1 format Null\n","query_status":"QueryFinish","tracestate":"another custom state","sorted_by_finish_time":1} {"query":"select * from url('http:\/\/127.0.0.2:8123\/?query=select%201%20format%20Null', CSV, 'a int')","query_status":"QueryFinish","tracestate":"another custom state","sorted_by_finish_time":1} -{"total spans":"3","unique spans":"3","unique non-zero parent spans":"3"} +{"total spans":"4","unique spans":"4","unique non-zero parent spans":"4"} {"initial query spans with proper parent":"1"} {"unique non-empty tracestate values":"1"} ===sampled=== diff --git a/tests/queries/0_stateless/01455_opentelemetry_distributed.sh b/tests/queries/0_stateless/01455_opentelemetry_distributed.sh index b2b5ae89105..0dfec6097db 100755 --- a/tests/queries/0_stateless/01455_opentelemetry_distributed.sh +++ b/tests/queries/0_stateless/01455_opentelemetry_distributed.sh @@ -12,6 +12,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) function check_log { ${CLICKHOUSE_CLIENT} --format=JSONEachRow -nq " +set allow_experimental_analyzer = 1; system flush logs; -- Show queries sorted by start time. @@ -55,7 +56,7 @@ select count(*) "'"'"initial query spans with proper parent"'"'" where trace_id = UUIDNumToString(toFixedString(unhex('$trace_id'), 16)) and operation_name = 'query' - and parent_span_id in ( + and parent_span_id in ( select span_id from system.opentelemetry_span_log where trace_id = UUIDNumToString(toFixedString(unhex('$trace_id'), 16)) and parent_span_id = reinterpretAsUInt64(unhex('73')) ) ; @@ -76,7 +77,7 @@ select uniqExact(value) "'"'"unique non-empty tracestate values"'"'" # Generate some random trace id so that the prevous runs of the test do not interfere. echo "===http===" -trace_id=$(${CLICKHOUSE_CLIENT} -q "select lower(hex(reverse(reinterpretAsString(generateUUIDv4()))))") +trace_id=$(${CLICKHOUSE_CLIENT} -q "select lower(hex(reverse(reinterpretAsString(generateUUIDv4())))) settings allow_experimental_analyzer = 1") # Check that the HTTP traceparent is read, and then passed through `remote` # table function. We expect 4 queries -- one initial, one SELECT and two @@ -86,7 +87,7 @@ ${CLICKHOUSE_CURL} \ --header "traceparent: 00-$trace_id-0000000000000073-01" \ --header "tracestate: some custom state" "$CLICKHOUSE_URL" \ --get \ - --data-urlencode "query=select 1 from remote('127.0.0.2', system, one) format Null" + --data-urlencode "query=select 1 from remote('127.0.0.2', system, one) settings allow_experimental_analyzer = 1 format Null" check_log diff --git a/tests/queries/0_stateless/01457_int256_hashing.sql b/tests/queries/0_stateless/01457_int256_hashing.sql index 861e567950a..510d25f6b8c 100644 --- a/tests/queries/0_stateless/01457_int256_hashing.sql +++ b/tests/queries/0_stateless/01457_int256_hashing.sql @@ -1,13 +1,15 @@ -- Tags: no-fasttest +SET joined_subquery_requires_alias = 0; + SELECT toUInt256(123) IN (NULL); SELECT toUInt256(123) AS k GROUP BY k; -SELECT toUInt256(123) AS k FROM system.one INNER JOIN (SELECT toUInt256(123) AS k) t USING k; +SELECT k FROM (SELECT toUInt256(123) AS k FROM system.one) INNER JOIN (SELECT toUInt256(123) AS k) t USING k; SELECT arrayEnumerateUniq([toUInt256(123), toUInt256(456), toUInt256(123)]); SELECT toInt256(123) IN (NULL); SELECT toInt256(123) AS k GROUP BY k; -SELECT toInt256(123) AS k FROM system.one INNER JOIN (SELECT toInt256(123) AS k) t USING k; +SELECT k FROM (SELECT toInt256(123) AS k FROM system.one) INNER JOIN (SELECT toInt256(123) AS k) t USING k; SELECT arrayEnumerateUniq([toInt256(123), toInt256(456), toInt256(123)]); -- SELECT toUInt128(123) IN (NULL); @@ -17,17 +19,17 @@ SELECT arrayEnumerateUniq([toInt256(123), toInt256(456), toInt256(123)]); SELECT toInt128(123) IN (NULL); SELECT toInt128(123) AS k GROUP BY k; -SELECT toInt128(123) AS k FROM system.one INNER JOIN (SELECT toInt128(123) AS k) t USING k; +SELECT k FROM (SELECT toInt128(123) AS k FROM system.one) INNER JOIN (SELECT toInt128(123) AS k) t USING k; SELECT arrayEnumerateUniq([toInt128(123), toInt128(456), toInt128(123)]); SELECT toNullable(toUInt256(321)) IN (NULL); SELECT toNullable(toUInt256(321)) AS k GROUP BY k; -SELECT toNullable(toUInt256(321)) AS k FROM system.one INNER JOIN (SELECT toUInt256(321) AS k) t USING k; +SELECT k FROM (SELECT toNullable(toUInt256(321)) AS k FROM system.one) INNER JOIN (SELECT toUInt256(321) AS k) t USING k; SELECT arrayEnumerateUniq([toNullable(toUInt256(321)), toNullable(toUInt256(456)), toNullable(toUInt256(321))]); SELECT toNullable(toInt256(321)) IN (NULL); SELECT toNullable(toInt256(321)) AS k GROUP BY k; -SELECT toNullable(toInt256(321)) AS k FROM system.one INNER JOIN (SELECT toInt256(321) AS k) t USING k; +SELECT k FROM (SELECT toNullable(toInt256(321)) AS k FROM system.one) INNER JOIN (SELECT toInt256(321) AS k) t USING k; SELECT arrayEnumerateUniq([toNullable(toInt256(321)), toNullable(toInt256(456)), toNullable(toInt256(321))]); -- SELECT toNullable(toUInt128(321)) IN (NULL); @@ -37,5 +39,5 @@ SELECT arrayEnumerateUniq([toNullable(toInt256(321)), toNullable(toInt256(456)), SELECT toNullable(toInt128(321)) IN (NULL); SELECT toNullable(toInt128(321)) AS k GROUP BY k; -SELECT toNullable(toInt128(321)) AS k FROM system.one INNER JOIN (SELECT toInt128(321) AS k) t USING k; +SELECT k FROM (SELECT toNullable(toInt128(321)) AS k FROM system.one) INNER JOIN (SELECT toInt128(321) AS k) t USING k; SELECT arrayEnumerateUniq([toNullable(toInt128(321)), toNullable(toInt128(456)), toNullable(toInt128(321))]); diff --git a/tests/queries/0_stateless/01470_columns_transformers.sql b/tests/queries/0_stateless/01470_columns_transformers.sql index 22c30ed36bf..8840ce3f3b5 100644 --- a/tests/queries/0_stateless/01470_columns_transformers.sql +++ b/tests/queries/0_stateless/01470_columns_transformers.sql @@ -1,5 +1,3 @@ -SET allow_experimental_analyzer = 1; - DROP TABLE IF EXISTS columns_transformers; CREATE TABLE columns_transformers (i Int64, j Int16, k Int64) Engine=TinyLog; @@ -19,15 +17,15 @@ SELECT a.* APPLY(toDate) EXCEPT(i, j) APPLY(any) from columns_transformers a; SELECT * EXCEPT STRICT i from columns_transformers; SELECT * EXCEPT STRICT (i, j) from columns_transformers; SELECT * EXCEPT STRICT i, j1 from columns_transformers; -- { serverError 47 } -SELECT * EXCEPT STRICT(i, j1) from columns_transformers; -- { serverError 36 } +SELECT * EXCEPT STRICT(i, j1) from columns_transformers; -- { serverError NO_SUCH_COLUMN_IN_TABLE , BAD_ARGUMENTS } SELECT * REPLACE STRICT i + 1 AS i from columns_transformers; -SELECT * REPLACE STRICT(i + 1 AS col) from columns_transformers; -- { serverError 36 } +SELECT * REPLACE STRICT(i + 1 AS col) from columns_transformers; -- { serverError NO_SUCH_COLUMN_IN_TABLE, BAD_ARGUMENTS } SELECT * REPLACE(i + 1 AS i) APPLY(sum) from columns_transformers; SELECT columns_transformers.* REPLACE(j + 2 AS j, i + 1 AS i) APPLY(avg) from columns_transformers; SELECT columns_transformers.* REPLACE(j + 1 AS j, j + 2 AS j) APPLY(avg) from columns_transformers; -- { serverError 43 } -- REPLACE after APPLY will not match anything SELECT a.* APPLY(toDate) REPLACE(i + 1 AS i) APPLY(any) from columns_transformers a; -SELECT a.* APPLY(toDate) REPLACE STRICT(i + 1 AS i) APPLY(any) from columns_transformers a; -- { serverError 36 } +SELECT a.* APPLY(toDate) REPLACE STRICT(i + 1 AS i) APPLY(any) from columns_transformers a; -- { serverError NO_SUCH_COLUMN_IN_TABLE, BAD_ARGUMENTS } EXPLAIN SYNTAX SELECT * APPLY(sum) from columns_transformers; EXPLAIN SYNTAX SELECT columns_transformers.* APPLY(avg) from columns_transformers; diff --git a/tests/queries/0_stateless/01474_bad_global_join.sql b/tests/queries/0_stateless/01474_bad_global_join.sql index 2251c1797cf..622e14e6f22 100644 --- a/tests/queries/0_stateless/01474_bad_global_join.sql +++ b/tests/queries/0_stateless/01474_bad_global_join.sql @@ -1,7 +1,5 @@ -- Tags: global -SET allow_experimental_analyzer = 1; - DROP TABLE IF EXISTS local_table; DROP TABLE IF EXISTS dist_table; @@ -12,7 +10,7 @@ INSERT INTO local_table SELECT number AS id, toString(number) AS val FROM number CREATE TABLE dist_table AS local_table ENGINE = Distributed('test_cluster_two_shards_localhost', currentDatabase(), local_table); -SELECT uniq(d.val) FROM dist_table AS d GLOBAL LEFT JOIN numbers(100) AS t USING id; -- { serverError 47 } +SELECT uniq(d.val) FROM dist_table AS d GLOBAL LEFT JOIN numbers(100) AS t USING id; -- { serverError 47, 284 } SELECT uniq(d.val) FROM dist_table AS d GLOBAL LEFT JOIN local_table AS t USING id; DROP TABLE local_table; diff --git a/tests/queries/0_stateless/01505_trivial_count_with_partition_predicate.sql b/tests/queries/0_stateless/01505_trivial_count_with_partition_predicate.sql index e4e2e3dd76a..e8643a4468c 100644 --- a/tests/queries/0_stateless/01505_trivial_count_with_partition_predicate.sql +++ b/tests/queries/0_stateless/01505_trivial_count_with_partition_predicate.sql @@ -7,16 +7,16 @@ insert into test1 values ('2020-09-01 00:01:02', 1), ('2020-09-01 20:01:03', 2), set max_rows_to_read = 1; -- non-optimized -select count() from test1 settings max_parallel_replicas = 3; -- { serverError 158; } +select count() from test1 settings max_parallel_replicas = 3; -- { serverError 158 } -- optimized (toYear is monotonic and we provide the partition expr as is) select count() from test1 where toYear(toDate(p)) = 1999; -- non-optimized (toDate(DateTime) is always monotonic, but we cannot relaxing the predicates to do trivial count()) -select count() from test1 where p > toDateTime('2020-09-01 10:00:00'); -- { serverError 158; } +select count() from test1 where p > toDateTime('2020-09-01 10:00:00'); -- { serverError 158 } -- optimized (partition expr wrapped with non-monotonic functions) select count() FROM test1 where toDate(p) = '2020-09-01' and sipHash64(toString(toDate(p))) % 2 = 1; select count() FROM test1 where toDate(p) = '2020-09-01' and sipHash64(toString(toDate(p))) % 2 = 0; -- non-optimized (some predicate depends on non-partition_expr columns) -select count() FROM test1 where toDate(p) = '2020-09-01' and k = 2; -- { serverError 158; } +select count() FROM test1 where toDate(p) = '2020-09-01' and k = 2; -- { serverError 158 } -- optimized select count() from test1 where toDate(p) > '2020-09-01'; -- non-optimized @@ -35,10 +35,10 @@ select count() from test_tuple where i > 2; -- optimized select count() from test_tuple where i < 1; -- non-optimized -select count() from test_tuple array join [p,p] as c where toDate(p) = '2020-09-01'; -- { serverError 158; } +select count() from test_tuple array join [p,p] as c where toDate(p) = '2020-09-01'; -- { serverError 158 } select count() from test_tuple array join [1,2] as c where toDate(p) = '2020-09-01' settings max_rows_to_read = 4; -- non-optimized -select count() from test_tuple array join [1,2,3] as c where toDate(p) = '2020-09-01'; -- { serverError 158; } +select count() from test_tuple array join [1,2,3] as c where toDate(p) = '2020-09-01'; -- { serverError 158 } select count() from test_tuple array join [1,2,3] as c where toDate(p) = '2020-09-01' settings max_rows_to_read = 6; create table test_two_args(i int, j int, k int) engine MergeTree partition by i + j order by k settings index_granularity = 1; @@ -48,7 +48,7 @@ insert into test_two_args values (1, 2, 3), (2, 1, 3), (0, 3, 4); -- optimized select count() from test_two_args where i + j = 3; -- non-optimized -select count() from test_two_args where i = 1; -- { serverError 158; } +select count() from test_two_args where i = 1; -- { serverError 158 } drop table test1; drop table test_tuple; diff --git a/tests/queries/0_stateless/01513_optimize_aggregation_in_order_memory_long.sql b/tests/queries/0_stateless/01513_optimize_aggregation_in_order_memory_long.sql index 228e4d73167..3d57518d0f4 100644 --- a/tests/queries/0_stateless/01513_optimize_aggregation_in_order_memory_long.sql +++ b/tests/queries/0_stateless/01513_optimize_aggregation_in_order_memory_long.sql @@ -13,9 +13,9 @@ set max_memory_usage='500M'; set max_threads=1; set max_block_size=500; -select key, groupArray(repeat('a', 200)), count() from data_01513 group by key format Null settings optimize_aggregation_in_order=0; -- { serverError 241; } +select key, groupArray(repeat('a', 200)), count() from data_01513 group by key format Null settings optimize_aggregation_in_order=0; -- { serverError 241 } select key, groupArray(repeat('a', 200)), count() from data_01513 group by key format Null settings optimize_aggregation_in_order=1; -- for WITH TOTALS previous groups should be kept. -select key, groupArray(repeat('a', 200)), count() from data_01513 group by key with totals format Null settings optimize_aggregation_in_order=1; -- { serverError 241; } +select key, groupArray(repeat('a', 200)), count() from data_01513 group by key with totals format Null settings optimize_aggregation_in_order=1; -- { serverError 241 } drop table data_01513; diff --git a/tests/queries/0_stateless/01516_create_table_primary_key.sql b/tests/queries/0_stateless/01516_create_table_primary_key.sql index b2b9f288eab..630c573c2cc 100644 --- a/tests/queries/0_stateless/01516_create_table_primary_key.sql +++ b/tests/queries/0_stateless/01516_create_table_primary_key.sql @@ -35,7 +35,7 @@ ATTACH TABLE primary_key_test(v1 Int32, v2 Int32) ENGINE=ReplacingMergeTree ORDE SELECT * FROM primary_key_test FINAL; DROP TABLE primary_key_test; -CREATE TABLE primary_key_test(v1 Int64, v2 Int32, v3 String, PRIMARY KEY(v1, gcd(v1, v2))) ENGINE=ReplacingMergeTree ORDER BY v1; -- { serverError 36; } +CREATE TABLE primary_key_test(v1 Int64, v2 Int32, v3 String, PRIMARY KEY(v1, gcd(v1, v2))) ENGINE=ReplacingMergeTree ORDER BY v1; -- { serverError 36 } CREATE TABLE primary_key_test(v1 Int64, v2 Int32, v3 String, PRIMARY KEY(v1, gcd(v1, v2))) ENGINE=ReplacingMergeTree ORDER BY (v1, gcd(v1, v2)); diff --git a/tests/queries/0_stateless/01527_bad_aggregation_in_lambda.sql b/tests/queries/0_stateless/01527_bad_aggregation_in_lambda.sql index e16f6396a2c..3be73ba56e7 100644 --- a/tests/queries/0_stateless/01527_bad_aggregation_in_lambda.sql +++ b/tests/queries/0_stateless/01527_bad_aggregation_in_lambda.sql @@ -1,3 +1 @@ -SET allow_experimental_analyzer = 1; - -SELECT arrayMap(x -> x * sum(x), range(10)); -- { serverError 10 } +SELECT arrayMap(x -> x * sum(x), range(10)); -- { serverError 10, 47 } diff --git a/tests/queries/0_stateless/01528_allow_nondeterministic_optimize_skip_unused_shards.sql b/tests/queries/0_stateless/01528_allow_nondeterministic_optimize_skip_unused_shards.sql index 08fba7480d1..ac04178e585 100644 --- a/tests/queries/0_stateless/01528_allow_nondeterministic_optimize_skip_unused_shards.sql +++ b/tests/queries/0_stateless/01528_allow_nondeterministic_optimize_skip_unused_shards.sql @@ -5,7 +5,7 @@ create table dist_01528 as system.one engine=Distributed('test_cluster_two_shard set optimize_skip_unused_shards=1; set force_optimize_skip_unused_shards=1; -select * from dist_01528 where dummy = 2; -- { serverError 507; } +select * from dist_01528 where dummy = 2; -- { serverError 507 } select * from dist_01528 where dummy = 2 settings allow_nondeterministic_optimize_skip_unused_shards=1; drop table dist_01528; diff --git a/tests/queries/0_stateless/01530_drop_database_atomic_sync.sql b/tests/queries/0_stateless/01530_drop_database_atomic_sync.sql index 7a2e64742cf..13b4a4e331b 100644 --- a/tests/queries/0_stateless/01530_drop_database_atomic_sync.sql +++ b/tests/queries/0_stateless/01530_drop_database_atomic_sync.sql @@ -30,7 +30,7 @@ create table db_01530_atomic.data (key Int) Engine=ReplicatedMergeTree('/clickho drop database db_01530_atomic; create database db_01530_atomic Engine=Atomic; -create table db_01530_atomic.data (key Int) Engine=ReplicatedMergeTree('/clickhouse/tables/{database}/db_01530_atomic/data', 'test') order by key; -- { serverError 253; } +create table db_01530_atomic.data (key Int) Engine=ReplicatedMergeTree('/clickhouse/tables/{database}/db_01530_atomic/data', 'test') order by key; -- { serverError 253 } set database_atomic_wait_for_drop_and_detach_synchronously=1; diff --git a/tests/queries/0_stateless/01551_mergetree_read_in_order_spread.sql b/tests/queries/0_stateless/01551_mergetree_read_in_order_spread.sql index e374e012238..c202ad349d6 100644 --- a/tests/queries/0_stateless/01551_mergetree_read_in_order_spread.sql +++ b/tests/queries/0_stateless/01551_mergetree_read_in_order_spread.sql @@ -1,4 +1,4 @@ --- Tags: no-s3-storage +-- Tags: no-s3-storage, no-random-merge-tree-settings DROP TABLE IF EXISTS data_01551; diff --git a/tests/queries/0_stateless/01555_system_distribution_queue_mask.reference b/tests/queries/0_stateless/01555_system_distribution_queue_mask.reference index 745160a517e..26aea1555a5 100644 --- a/tests/queries/0_stateless/01555_system_distribution_queue_mask.reference +++ b/tests/queries/0_stateless/01555_system_distribution_queue_mask.reference @@ -1,3 +1,5 @@ +masked flush only +3,"default:*@127%2E0%2E0%2E1:9000,default:*@127%2E0%2E0%2E2:9000","AUTHENTICATION_FAILED",1 masked 3,"default:*@127%2E0%2E0%2E1:9000,default:*@127%2E0%2E0%2E2:9000","AUTHENTICATION_FAILED",1 no masking diff --git a/tests/queries/0_stateless/01555_system_distribution_queue_mask.sql b/tests/queries/0_stateless/01555_system_distribution_queue_mask.sql index fea75e1439f..61083c3ae14 100644 --- a/tests/queries/0_stateless/01555_system_distribution_queue_mask.sql +++ b/tests/queries/0_stateless/01555_system_distribution_queue_mask.sql @@ -9,6 +9,20 @@ drop table if exists dist_01555; drop table if exists data_01555; create table data_01555 (key Int) Engine=Null(); +-- +-- masked flush only +-- +SELECT 'masked flush only'; +create table dist_01555 (key Int) Engine=Distributed(test_cluster_with_incorrect_pw, currentDatabase(), data_01555, key); +system stop distributed sends dist_01555; + +insert into dist_01555 values (1)(2); +-- since test_cluster_with_incorrect_pw contains incorrect password ignore error +system flush distributed dist_01555; -- { serverError 516 } +select length(splitByChar('*', data_path)), replaceRegexpOne(data_path, '^.*/([^/]*)/' , '\\1'), extract(last_exception, 'AUTHENTICATION_FAILED'), dateDiff('s', last_exception_time, now()) < 5 from system.distribution_queue where database = currentDatabase() and table = 'dist_01555' format CSV; + +drop table dist_01555; + -- -- masked -- @@ -17,7 +31,7 @@ create table dist_01555 (key Int) Engine=Distributed(test_cluster_with_incorrect insert into dist_01555 values (1)(2); -- since test_cluster_with_incorrect_pw contains incorrect password ignore error -system flush distributed dist_01555; -- { serverError 516; } +system flush distributed dist_01555; -- { serverError 516 } select length(splitByChar('*', data_path)), replaceRegexpOne(data_path, '^.*/([^/]*)/' , '\\1'), extract(last_exception, 'AUTHENTICATION_FAILED'), dateDiff('s', last_exception_time, now()) < 5 from system.distribution_queue where database = currentDatabase() and table = 'dist_01555' format CSV; drop table dist_01555; @@ -29,7 +43,6 @@ SELECT 'no masking'; create table dist_01555 (key Int) Engine=Distributed(test_shard_localhost, currentDatabase(), data_01555, key); insert into dist_01555 values (1)(2); --- since test_cluster_with_incorrect_pw contains incorrect password ignore error system flush distributed dist_01555; select length(splitByChar('*', data_path)), replaceRegexpOne(data_path, '^.*/([^/]*)/' , '\\1') from system.distribution_queue where database = currentDatabase() and table = 'dist_01555' format CSV; diff --git a/tests/queries/0_stateless/01556_explain_select_with_union_query.reference b/tests/queries/0_stateless/01556_explain_select_with_union_query.reference index c18e6b70b0d..27cf4c129b1 100644 --- a/tests/queries/0_stateless/01556_explain_select_with_union_query.reference +++ b/tests/queries/0_stateless/01556_explain_select_with_union_query.reference @@ -1,180 +1,180 @@ Union - Expression ((Projection + Before ORDER BY)) + Expression ((Project names + (Projection + Change column names to column identifiers))) ReadFromStorage (SystemOne) - Expression ((Projection + Before ORDER BY)) + Expression ((Project names + (Projection + Change column names to column identifiers))) ReadFromStorage (SystemOne) - Expression ((Projection + Before ORDER BY)) + Expression ((Project names + (Projection + Change column names to column identifiers))) ReadFromStorage (SystemOne) Union - Expression ((Projection + Before ORDER BY)) + Expression ((Project names + (Projection + Change column names to column identifiers))) ReadFromStorage (SystemOne) - Expression ((Projection + Before ORDER BY)) + Expression ((Project names + (Projection + Change column names to column identifiers))) ReadFromStorage (SystemOne) - Expression ((Projection + Before ORDER BY)) + Expression ((Project names + (Projection + Change column names to column identifiers))) ReadFromStorage (SystemOne) Distinct Union - Expression ((Projection + Before ORDER BY)) + Expression ((Project names + (Projection + Change column names to column identifiers))) ReadFromStorage (SystemOne) - Expression ((Projection + Before ORDER BY)) + Expression ((Project names + (Projection + Change column names to column identifiers))) ReadFromStorage (SystemOne) - Expression ((Projection + Before ORDER BY)) + Expression ((Project names + (Projection + Change column names to column identifiers))) ReadFromStorage (SystemOne) Distinct Union - Expression ((Projection + Before ORDER BY)) + Expression ((Project names + (Projection + Change column names to column identifiers))) ReadFromStorage (SystemOne) - Expression ((Projection + Before ORDER BY)) + Expression ((Project names + (Projection + Change column names to column identifiers))) ReadFromStorage (SystemOne) - Expression ((Projection + Before ORDER BY)) + Expression ((Project names + (Projection + Change column names to column identifiers))) ReadFromStorage (SystemOne) Distinct Union - Expression ((Projection + Before ORDER BY)) + Expression ((Project names + (Projection + Change column names to column identifiers))) ReadFromStorage (SystemOne) - Expression ((Projection + Before ORDER BY)) + Expression ((Project names + (Projection + Change column names to column identifiers))) ReadFromStorage (SystemOne) - Expression ((Projection + Before ORDER BY)) + Expression ((Project names + (Projection + Change column names to column identifiers))) ReadFromStorage (SystemOne) Distinct Union - Expression ((Projection + Before ORDER BY)) + Expression ((Project names + (Projection + Change column names to column identifiers))) ReadFromStorage (SystemOne) - Expression ((Projection + Before ORDER BY)) + Expression ((Project names + (Projection + Change column names to column identifiers))) ReadFromStorage (SystemOne) - Expression ((Projection + Before ORDER BY)) + Expression ((Project names + (Projection + Change column names to column identifiers))) ReadFromStorage (SystemOne) Union - Expression ((Projection + Before ORDER BY)) + Expression ((Project names + (Projection + Change column names to column identifiers))) ReadFromStorage (SystemOne) - Expression ((Projection + Before ORDER BY)) + Expression ((Project names + (Projection + Change column names to column identifiers))) ReadFromStorage (SystemOne) Distinct Union - Expression ((Projection + Before ORDER BY)) + Expression ((Project names + (Projection + Change column names to column identifiers))) ReadFromStorage (SystemOne) - Expression ((Projection + Before ORDER BY)) + Expression ((Project names + (Projection + Change column names to column identifiers))) ReadFromStorage (SystemOne) - Expression ((Projection + Before ORDER BY)) + Expression ((Project names + (Projection + Change column names to column identifiers))) ReadFromStorage (SystemOne) Distinct Union - Expression ((Projection + Before ORDER BY)) + Expression ((Project names + (Projection + Change column names to column identifiers))) ReadFromStorage (SystemOne) - Expression ((Projection + Before ORDER BY)) + Expression ((Project names + (Projection + Change column names to column identifiers))) ReadFromStorage (SystemOne) - Expression ((Projection + Before ORDER BY)) + Expression ((Project names + (Projection + Change column names to column identifiers))) ReadFromStorage (SystemOne) - Expression ((Projection + Before ORDER BY)) + Expression ((Project names + (Projection + Change column names to column identifiers))) ReadFromStorage (SystemOne) - Expression ((Projection + Before ORDER BY)) + Expression ((Project names + (Projection + Change column names to column identifiers))) ReadFromStorage (SystemOne) - Expression ((Projection + Before ORDER BY)) + Expression ((Project names + (Projection + Change column names to column identifiers))) ReadFromStorage (SystemOne) Distinct Union - Expression ((Projection + Before ORDER BY)) + Expression ((Project names + (Projection + Change column names to column identifiers))) ReadFromStorage (SystemOne) - Expression ((Projection + Before ORDER BY)) + Expression ((Project names + (Projection + Change column names to column identifiers))) ReadFromStorage (SystemOne) - Expression ((Projection + Before ORDER BY)) + Expression ((Project names + (Projection + Change column names to column identifiers))) ReadFromStorage (SystemOne) Distinct Union - Expression ((Projection + Before ORDER BY)) + Expression ((Project names + (Projection + Change column names to column identifiers))) ReadFromStorage (SystemOne) - Expression ((Projection + Before ORDER BY)) + Expression ((Project names + (Projection + Change column names to column identifiers))) ReadFromStorage (SystemOne) Distinct Union - Expression ((Projection + Before ORDER BY)) + Expression ((Project names + (Projection + Change column names to column identifiers))) ReadFromStorage (SystemOne) - Expression ((Projection + Before ORDER BY)) + Expression ((Project names + (Projection + Change column names to column identifiers))) ReadFromStorage (SystemOne) Union - Expression ((Projection + Before ORDER BY)) + Expression ((Project names + (Projection + Change column names to column identifiers))) ReadFromStorage (SystemOne) - Expression ((Projection + Before ORDER BY)) + Expression ((Project names + (Projection + Change column names to column identifiers))) ReadFromStorage (SystemOne) - Expression ((Projection + Before ORDER BY)) + Expression ((Project names + (Projection + Change column names to column identifiers))) ReadFromStorage (SystemOne) Union - Expression ((Projection + Before ORDER BY)) + Expression ((Project names + (Projection + Change column names to column identifiers))) ReadFromStorage (SystemOne) - Expression ((Projection + Before ORDER BY)) + Expression ((Project names + (Projection + Change column names to column identifiers))) ReadFromStorage (SystemOne) - Expression ((Projection + Before ORDER BY)) + Expression ((Project names + (Projection + Change column names to column identifiers))) ReadFromStorage (SystemOne) Union - Expression ((Projection + Before ORDER BY)) + Expression ((Project names + (Projection + Change column names to column identifiers))) ReadFromStorage (SystemOne) - Expression ((Projection + Before ORDER BY)) + Expression ((Project names + (Projection + Change column names to column identifiers))) ReadFromStorage (SystemOne) - Expression ((Projection + Before ORDER BY)) + Expression ((Project names + (Projection + Change column names to column identifiers))) ReadFromStorage (SystemOne) Distinct Union - Expression ((Projection + Before ORDER BY)) + Expression ((Project names + (Projection + Change column names to column identifiers))) ReadFromStorage (SystemOne) - Expression ((Projection + Before ORDER BY)) + Expression ((Project names + (Projection + Change column names to column identifiers))) ReadFromStorage (SystemOne) - Expression ((Projection + Before ORDER BY)) + Expression ((Project names + (Projection + Change column names to column identifiers))) ReadFromStorage (SystemOne) Distinct Union - Expression ((Projection + Before ORDER BY)) + Expression ((Project names + (Projection + Change column names to column identifiers))) ReadFromStorage (SystemOne) - Expression ((Projection + Before ORDER BY)) + Expression ((Project names + (Projection + Change column names to column identifiers))) ReadFromStorage (SystemOne) - Expression ((Projection + Before ORDER BY)) + Expression ((Project names + (Projection + Change column names to column identifiers))) ReadFromStorage (SystemOne) Distinct Union - Expression ((Projection + Before ORDER BY)) + Expression ((Project names + (Projection + Change column names to column identifiers))) ReadFromStorage (SystemOne) - Expression ((Projection + Before ORDER BY)) + Expression ((Project names + (Projection + Change column names to column identifiers))) ReadFromStorage (SystemOne) - Expression ((Projection + Before ORDER BY)) + Expression ((Project names + (Projection + Change column names to column identifiers))) ReadFromStorage (SystemOne) Union - Expression ((Projection + Before ORDER BY)) + Expression ((Project names + (Projection + Change column names to column identifiers))) ReadFromStorage (SystemOne) - Expression ((Projection + Before ORDER BY)) + Expression ((Project names + (Projection + Change column names to column identifiers))) ReadFromStorage (SystemOne) - Expression ((Projection + Before ORDER BY)) + Expression ((Project names + (Projection + Change column names to column identifiers))) ReadFromStorage (SystemOne) - Expression ((Projection + Before ORDER BY)) + Expression ((Project names + (Projection + Change column names to column identifiers))) ReadFromStorage (SystemOne) - Expression ((Projection + Before ORDER BY)) + Expression ((Project names + (Projection + Change column names to column identifiers))) ReadFromStorage (SystemOne) - Expression ((Projection + Before ORDER BY)) + Expression ((Project names + (Projection + Change column names to column identifiers))) ReadFromStorage (SystemOne) - Expression ((Projection + Before ORDER BY)) + Expression ((Project names + (Projection + Change column names to column identifiers))) ReadFromStorage (SystemOne) Distinct Union - Expression ((Projection + Before ORDER BY)) + Expression ((Project names + (Projection + Change column names to column identifiers))) ReadFromStorage (SystemOne) - Expression ((Projection + Before ORDER BY)) + Expression ((Project names + (Projection + Change column names to column identifiers))) ReadFromStorage (SystemOne) - Expression ((Projection + Before ORDER BY)) + Expression ((Project names + (Projection + Change column names to column identifiers))) ReadFromStorage (SystemOne) - Expression ((Projection + Before ORDER BY)) + Expression ((Project names + (Projection + Change column names to column identifiers))) ReadFromStorage (SystemOne) Union - Expression ((Projection + Before ORDER BY)) + Expression ((Project names + (Projection + Change column names to column identifiers))) ReadFromStorage (SystemOne) - Expression ((Projection + Before ORDER BY)) + Expression ((Project names + (Projection + Change column names to column identifiers))) ReadFromStorage (SystemOne) - Expression ((Projection + Before ORDER BY)) + Expression ((Project names + (Projection + Change column names to column identifiers))) ReadFromStorage (SystemOne) Union - Expression ((Projection + Before ORDER BY)) + Expression ((Project names + (Projection + Change column names to column identifiers))) ReadFromStorage (SystemOne) - Expression ((Projection + Before ORDER BY)) + Expression ((Project names + (Projection + Change column names to column identifiers))) ReadFromStorage (SystemOne) Union - Expression ((Projection + Before ORDER BY)) + Expression ((Project names + (Projection + Change column names to column identifiers))) ReadFromStorage (SystemOne) - Expression ((Projection + Before ORDER BY)) + Expression ((Project names + (Projection + Change column names to column identifiers))) ReadFromStorage (SystemOne) diff --git a/tests/queries/0_stateless/01556_explain_select_with_union_query.sql b/tests/queries/0_stateless/01556_explain_select_with_union_query.sql index dcd9bbe7347..bbd96ef5c69 100644 --- a/tests/queries/0_stateless/01556_explain_select_with_union_query.sql +++ b/tests/queries/0_stateless/01556_explain_select_with_union_query.sql @@ -1,3 +1,4 @@ +SET allow_experimental_analyzer = 1; SET union_default_mode = 'DISTINCT'; set enable_global_with_statement = 1; diff --git a/tests/queries/0_stateless/01561_clickhouse_client_stage.reference b/tests/queries/0_stateless/01561_clickhouse_client_stage.reference index 44c39f2a444..00e0f4ddb2e 100644 --- a/tests/queries/0_stateless/01561_clickhouse_client_stage.reference +++ b/tests/queries/0_stateless/01561_clickhouse_client_stage.reference @@ -1,15 +1,15 @@ -execute: default +execute: --allow_experimental_analyzer=1 "foo" 1 -execute: --stage fetch_columns -"dummy" +execute: --allow_experimental_analyzer=1 --stage fetch_columns +"system.one.dummy_0" 0 -execute: --stage with_mergeable_state -"1" +execute: --allow_experimental_analyzer=1 --stage with_mergeable_state +"1_UInt8" 1 -execute: --stage with_mergeable_state_after_aggregation -"1" +execute: --allow_experimental_analyzer=1 --stage with_mergeable_state_after_aggregation +"1_UInt8" 1 -execute: --stage complete +execute: --allow_experimental_analyzer=1 --stage complete "foo" 1 diff --git a/tests/queries/0_stateless/01561_clickhouse_client_stage.sh b/tests/queries/0_stateless/01561_clickhouse_client_stage.sh index 72161333812..99267458421 100755 --- a/tests/queries/0_stateless/01561_clickhouse_client_stage.sh +++ b/tests/queries/0_stateless/01561_clickhouse_client_stage.sh @@ -5,6 +5,10 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh +opts=( + "--allow_experimental_analyzer=1" +) + function execute_query() { if [ $# -eq 0 ]; then @@ -15,8 +19,8 @@ function execute_query() ${CLICKHOUSE_CLIENT} "$@" --format CSVWithNames -q "SELECT 1 AS foo" } -execute_query # default -- complete -execute_query --stage fetch_columns -execute_query --stage with_mergeable_state -execute_query --stage with_mergeable_state_after_aggregation -execute_query --stage complete +execute_query "${opts[@]}" # default -- complete +execute_query "${opts[@]}" --stage fetch_columns +execute_query "${opts[@]}" --stage with_mergeable_state +execute_query "${opts[@]}" --stage with_mergeable_state_after_aggregation +execute_query "${opts[@]}" --stage complete diff --git a/tests/queries/0_stateless/01562_optimize_monotonous_functions_in_order_by.reference b/tests/queries/0_stateless/01562_optimize_monotonous_functions_in_order_by.reference index 8c8bb73b801..49b4d22ea71 100644 --- a/tests/queries/0_stateless/01562_optimize_monotonous_functions_in_order_by.reference +++ b/tests/queries/0_stateless/01562_optimize_monotonous_functions_in_order_by.reference @@ -4,10 +4,10 @@ SELECT FROM test_order_by ORDER BY timestamp ASC LIMIT 10 -Expression (Projection) +Expression (Project names) Limit (preliminary LIMIT (without OFFSET)) Sorting (Sorting for ORDER BY) - Expression (Before ORDER BY) + Expression ((Before ORDER BY + (Projection + Change column names to column identifiers))) ReadFromMergeTree (default.test_order_by) SELECT timestamp, @@ -15,10 +15,10 @@ SELECT FROM test_order_by ORDER BY toDate(timestamp) ASC LIMIT 10 -Expression (Projection) +Expression (Project names) Limit (preliminary LIMIT (without OFFSET)) Sorting (Sorting for ORDER BY) - Expression (Before ORDER BY) + Expression ((Before ORDER BY + (Projection + Change column names to column identifiers))) ReadFromMergeTree (default.test_order_by) SELECT timestamp, @@ -28,10 +28,10 @@ ORDER BY toDate(timestamp) ASC, timestamp ASC LIMIT 10 -Expression (Projection) +Expression (Project names) Limit (preliminary LIMIT (without OFFSET)) Sorting (Sorting for ORDER BY) - Expression (Before ORDER BY) + Expression ((Before ORDER BY + (Projection + Change column names to column identifiers))) ReadFromMergeTree (default.test_order_by) SELECT timestamp, diff --git a/tests/queries/0_stateless/01562_optimize_monotonous_functions_in_order_by.sql b/tests/queries/0_stateless/01562_optimize_monotonous_functions_in_order_by.sql index 15ddb5a848f..2fe24d1662d 100644 --- a/tests/queries/0_stateless/01562_optimize_monotonous_functions_in_order_by.sql +++ b/tests/queries/0_stateless/01562_optimize_monotonous_functions_in_order_by.sql @@ -1,3 +1,4 @@ +SET allow_experimental_analyzer = 1; SET optimize_monotonous_functions_in_order_by = 1; SET optimize_read_in_order = 1; diff --git a/tests/queries/0_stateless/01591_window_functions.reference b/tests/queries/0_stateless/01591_window_functions.reference index aaa88d66ca0..b981a46b4fd 100644 --- a/tests/queries/0_stateless/01591_window_functions.reference +++ b/tests/queries/0_stateless/01591_window_functions.reference @@ -915,12 +915,12 @@ from (select number, intDiv(number, 3) p, mod(number, 5) o from numbers(16)) t ; -Expression ((Projection + Before ORDER BY)) +Expression ((Project names + Projection)) Window (Window step for window \'\') - Window (Window step for window \'PARTITION BY p\') - Window (Window step for window \'PARTITION BY p ORDER BY o ASC\') - Sorting (Sorting for window \'PARTITION BY p ORDER BY o ASC\') - Expression ((Before window functions + (Projection + Before ORDER BY))) + Window (Window step for window \'PARTITION BY t.p_0\') + Window (Window step for window \'PARTITION BY t.p_0 ORDER BY t.o_1 ASC\') + Sorting (Sorting for window \'PARTITION BY t.p_0 ORDER BY t.o_1 ASC\') + Expression ((Before WINDOW + (Change column names to column identifiers + (Project names + (Projection + Change column names to column identifiers))))) ReadFromStorage (SystemNumbers) explain select count(*) over (order by o, number), @@ -929,13 +929,13 @@ from (select number, intDiv(number, 3) p, mod(number, 5) o from numbers(16)) t ; -Expression ((Projection + Before ORDER BY)) - Window (Window step for window \'ORDER BY o ASC, number ASC\') - Sorting (Sorting for window \'ORDER BY o ASC, number ASC\') - Window (Window step for window \'ORDER BY number ASC\') - Expression ((Before window functions + (Projection + Before ORDER BY)) [lifted up part]) - Sorting (Sorting for window \'ORDER BY number ASC\') - Expression ((Before window functions + (Projection + Before ORDER BY))) +Expression ((Project names + Projection)) + Window (Window step for window \'ORDER BY t.o_0 ASC, t.number_1 ASC\') + Sorting (Sorting for window \'ORDER BY t.o_0 ASC, t.number_1 ASC\') + Window (Window step for window \'ORDER BY t.number_1 ASC\') + Expression ((Before WINDOW + (Change column names to column identifiers + (Project names + (Projection + Change column names to column identifiers)))) [lifted up part]) + Sorting (Sorting for window \'ORDER BY t.number_1 ASC\') + Expression ((Before WINDOW + (Change column names to column identifiers + (Project names + (Projection + Change column names to column identifiers))))) ReadFromStorage (SystemNumbers) -- A test case for the sort comparator found by fuzzer. SELECT diff --git a/tests/queries/0_stateless/01591_window_functions.sql b/tests/queries/0_stateless/01591_window_functions.sql index 3f4a028eac2..3c9c1f9cea7 100644 --- a/tests/queries/0_stateless/01591_window_functions.sql +++ b/tests/queries/0_stateless/01591_window_functions.sql @@ -1,5 +1,7 @@ -- Tags: long +SET allow_experimental_analyzer = 1; + -- { echo } -- just something basic diff --git a/tests/queries/0_stateless/01592_toUnixTimestamp_Date.sql b/tests/queries/0_stateless/01592_toUnixTimestamp_Date.sql index 5dc87e31f75..e8411484d71 100644 --- a/tests/queries/0_stateless/01592_toUnixTimestamp_Date.sql +++ b/tests/queries/0_stateless/01592_toUnixTimestamp_Date.sql @@ -1 +1 @@ -select toUnixTimestamp(today()); -- { serverError 44; } +select toUnixTimestamp(today()); -- { serverError 44 } diff --git a/tests/queries/0_stateless/01595_countMatches.sql b/tests/queries/0_stateless/01595_countMatches.sql index 6374fe7bc5b..0b170945d44 100644 --- a/tests/queries/0_stateless/01595_countMatches.sql +++ b/tests/queries/0_stateless/01595_countMatches.sql @@ -25,5 +25,5 @@ select countMatchesCaseInsensitive('foo.com BAR.COM baz.com bam.com', '([^. ]+)\ select countMatchesCaseInsensitive('foo.com@foo.com bar.com@foo.com BAZ.com@foo.com bam.com@foo.com', '([^. ]+)\.([^. ]+)@([^. ]+)\.([^. ]+)'); select 'errors'; -select countMatches(1, 'foo') from numbers(1); -- { serverError 43; } -select countMatches('foobarfoo', toString(number)) from numbers(1); -- { serverError 44; } +select countMatches(1, 'foo') from numbers(1); -- { serverError 43 } +select countMatches('foobarfoo', toString(number)) from numbers(1); -- { serverError 44 } diff --git a/tests/queries/0_stateless/01596_full_join_chertus.sql b/tests/queries/0_stateless/01596_full_join_chertus.sql index 162b9f7be8f..32911abb792 100644 --- a/tests/queries/0_stateless/01596_full_join_chertus.sql +++ b/tests/queries/0_stateless/01596_full_join_chertus.sql @@ -1,9 +1,9 @@ select toTypeName(materialize(js1.k)), toTypeName(materialize(js2.k)), toTypeName(materialize(js1.s)), toTypeName(materialize(js2.s)) from (select number k, toLowCardinality(toString(number)) s from numbers(2)) as js1 full join (select toLowCardinality(number+1) k, toString(number+1) s from numbers(2)) as js2 -using k order by js1.k, js2.k; +ON js1.k = js2.k order by js1.k, js2.k; select toTypeName(js1.k), toTypeName(js2.k), toTypeName(js1.s), toTypeName(js2.s) from (select number k, toLowCardinality(toString(number)) s from numbers(2)) as js1 full join (select toLowCardinality(number+1) k, toString(number+1) s from numbers(2)) as js2 -using k order by js1.k, js2.k; +ON js1.k = js2.k order by js1.k, js2.k; diff --git a/tests/queries/0_stateless/01647_clickhouse_local_hung.sh b/tests/queries/0_stateless/01647_clickhouse_local_hung.sh index 04f32055ab6..4789db18b2e 100755 --- a/tests/queries/0_stateless/01647_clickhouse_local_hung.sh +++ b/tests/queries/0_stateless/01647_clickhouse_local_hung.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# Tags: no-fasttest set -e diff --git a/tests/queries/0_stateless/01674_where_prewhere_array_crash.sql b/tests/queries/0_stateless/01674_where_prewhere_array_crash.sql index 7a2466c70d7..98094f06509 100644 --- a/tests/queries/0_stateless/01674_where_prewhere_array_crash.sql +++ b/tests/queries/0_stateless/01674_where_prewhere_array_crash.sql @@ -1,7 +1,5 @@ -SET allow_experimental_analyzer = 1; - drop table if exists tab; create table tab (x UInt64, `arr.a` Array(UInt64), `arr.b` Array(UInt64)) engine = MergeTree order by x; -select x from tab array join arr prewhere x != 0 where arr; -- { serverError 43 } -select x from tab array join arr prewhere arr where x != 0; -- { serverError 43 } +select x from tab array join arr prewhere x != 0 where arr; -- { serverError 43, 47 } +select x from tab array join arr prewhere arr where x != 0; -- { serverError 43, 47 } drop table if exists tab; diff --git a/tests/queries/0_stateless/01705_normalize_create_alter_function_names.reference b/tests/queries/0_stateless/01705_normalize_create_alter_function_names.reference index b6f5fe99ca1..b5b93c34c00 100644 --- a/tests/queries/0_stateless/01705_normalize_create_alter_function_names.reference +++ b/tests/queries/0_stateless/01705_normalize_create_alter_function_names.reference @@ -1,2 +1,2 @@ -CREATE TABLE default.x\n(\n `i` Int32,\n INDEX mm rand() TYPE minmax GRANULARITY 1,\n INDEX nn rand() TYPE minmax GRANULARITY 1,\n PROJECTION p\n (\n SELECT max(i)\n ),\n PROJECTION p2\n (\n SELECT min(i)\n )\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/default/x\', \'r\')\nORDER BY i\nSETTINGS index_granularity = 8192 -metadata format version: 1\ndate column: \nsampling expression: \nindex granularity: 8192\nmode: 0\nsign column: \nprimary key: i\ndata format version: 1\npartition key: \nindices: mm rand() TYPE minmax GRANULARITY 1, nn rand() TYPE minmax GRANULARITY 1\nprojections: p (SELECT max(i)), p2 (SELECT min(i))\ngranularity bytes: 10485760\n +CREATE TABLE default.x\n(\n `i` Int32,\n INDEX mm log2(i) TYPE minmax GRANULARITY 1,\n INDEX nn log2(i) TYPE minmax GRANULARITY 1,\n PROJECTION p\n (\n SELECT max(i)\n ),\n PROJECTION p2\n (\n SELECT min(i)\n )\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/default/x\', \'r\')\nORDER BY i\nSETTINGS index_granularity = 8192 +metadata format version: 1\ndate column: \nsampling expression: \nindex granularity: 8192\nmode: 0\nsign column: \nprimary key: i\ndata format version: 1\npartition key: \nindices: mm log2(i) TYPE minmax GRANULARITY 1, nn log2(i) TYPE minmax GRANULARITY 1\nprojections: p (SELECT max(i)), p2 (SELECT min(i))\ngranularity bytes: 10485760\n diff --git a/tests/queries/0_stateless/01705_normalize_create_alter_function_names.sql b/tests/queries/0_stateless/01705_normalize_create_alter_function_names.sql index 683bd271405..be0f7e8b710 100644 --- a/tests/queries/0_stateless/01705_normalize_create_alter_function_names.sql +++ b/tests/queries/0_stateless/01705_normalize_create_alter_function_names.sql @@ -2,9 +2,9 @@ drop table if exists x; -create table x(i int, index mm RAND() type minmax granularity 1, projection p (select MAX(i))) engine ReplicatedMergeTree('/clickhouse/tables/{database}/x', 'r') order by i; +create table x(i int, index mm LOG2(i) type minmax granularity 1, projection p (select MAX(i))) engine ReplicatedMergeTree('/clickhouse/tables/{database}/x', 'r') order by i; -alter table x add index nn RAND() type minmax granularity 1, add projection p2 (select MIN(i)); +alter table x add index nn LOG2(i) type minmax granularity 1, add projection p2 (select MIN(i)); show create x; diff --git a/tests/queries/0_stateless/01709_inactive_parts_to_throw_insert.sql b/tests/queries/0_stateless/01709_inactive_parts_to_throw_insert.sql index 6de0d4f4e0c..2bb92aec713 100644 --- a/tests/queries/0_stateless/01709_inactive_parts_to_throw_insert.sql +++ b/tests/queries/0_stateless/01709_inactive_parts_to_throw_insert.sql @@ -7,6 +7,6 @@ insert into data_01709 values (2); optimize table data_01709 final; -insert into data_01709 values (3); -- { serverError 252; } +insert into data_01709 values (3); -- { serverError 252 } drop table data_01709; diff --git a/tests/queries/0_stateless/01710_projection_optimize_materialize.sql b/tests/queries/0_stateless/01710_projection_optimize_materialize.sql index d8251aabaf6..92d3ead828c 100644 --- a/tests/queries/0_stateless/01710_projection_optimize_materialize.sql +++ b/tests/queries/0_stateless/01710_projection_optimize_materialize.sql @@ -1,6 +1,6 @@ drop table if exists z; -create table z (pk Int64, d Date, id UInt64, c UInt64) Engine MergeTree partition by d order by pk ; +create table z (pk Int64, d Date, id UInt64, c UInt64) Engine MergeTree partition by d order by pk settings ratio_of_defaults_for_sparse_serialization = 1.0; insert into z select number, '2021-10-24', intDiv (number, 10000), 1 from numbers(1000000); optimize table z final; diff --git a/tests/queries/0_stateless/01710_projection_with_mixed_pipeline.sql b/tests/queries/0_stateless/01710_projection_with_mixed_pipeline.sql index 734aa659146..5169c667b81 100644 --- a/tests/queries/0_stateless/01710_projection_with_mixed_pipeline.sql +++ b/tests/queries/0_stateless/01710_projection_with_mixed_pipeline.sql @@ -4,6 +4,6 @@ create table t (x UInt32) engine = MergeTree order by tuple() settings index_gra insert into t select number from numbers(100); alter table t add projection p (select uniqHLL12(x)); insert into t select number + 100 from numbers(100); -select uniqHLL12(x) from t settings allow_experimental_projection_optimization = 1, max_bytes_to_read=400, max_block_size=8; -- { serverError 307; } +select uniqHLL12(x) from t settings allow_experimental_projection_optimization = 1, max_bytes_to_read=400, max_block_size=8; -- { serverError 307 } drop table if exists t; diff --git a/tests/queries/0_stateless/01710_projections.sql b/tests/queries/0_stateless/01710_projections.sql index cbabd3ec598..5097a88c8fa 100644 --- a/tests/queries/0_stateless/01710_projections.sql +++ b/tests/queries/0_stateless/01710_projections.sql @@ -1,6 +1,6 @@ drop table if exists projection_test; -create table projection_test (`sum(block_count)` UInt64, domain_alias UInt64 alias length(domain), datetime DateTime, domain LowCardinality(String), x_id String, y_id String, block_count Int64, retry_count Int64, duration Int64, kbytes Int64, buffer_time Int64, first_time Int64, total_bytes Nullable(UInt64), valid_bytes Nullable(UInt64), completed_bytes Nullable(UInt64), fixed_bytes Nullable(UInt64), force_bytes Nullable(UInt64), projection p (select toStartOfMinute(datetime) dt_m, countIf(first_time = 0) / count(), avg((kbytes * 8) / duration), count(), sum(block_count) / sum(duration), avg(block_count / duration), sum(buffer_time) / sum(duration), avg(buffer_time / duration), sum(valid_bytes) / sum(total_bytes), sum(completed_bytes) / sum(total_bytes), sum(fixed_bytes) / sum(total_bytes), sum(force_bytes) / sum(total_bytes), sum(valid_bytes) / sum(total_bytes), sum(retry_count) / sum(duration), avg(retry_count / duration), countIf(block_count > 0) / count(), countIf(first_time = 0) / count(), uniqHLL12(x_id), uniqHLL12(y_id) group by dt_m, domain)) engine MergeTree partition by toDate(datetime) order by (toStartOfTenMinutes(datetime), domain); +create table projection_test (`sum(block_count)` UInt64, domain_alias UInt64 alias length(domain), datetime DateTime, domain LowCardinality(String), x_id String, y_id String, block_count Int64, retry_count Int64, duration Int64, kbytes Int64, buffer_time Int64, first_time Int64, total_bytes Nullable(UInt64), valid_bytes Nullable(UInt64), completed_bytes Nullable(UInt64), fixed_bytes Nullable(UInt64), force_bytes Nullable(UInt64), projection p (select toStartOfMinute(datetime) dt_m, countIf(first_time = 0) / count(), avg((kbytes * 8) / duration), count(), sum(block_count) / sum(duration), avg(block_count / duration), sum(buffer_time) / sum(duration), avg(buffer_time / duration), sum(valid_bytes) / sum(total_bytes), sum(completed_bytes) / sum(total_bytes), sum(fixed_bytes) / sum(total_bytes), sum(force_bytes) / sum(total_bytes), sum(valid_bytes) / sum(total_bytes), sum(retry_count) / sum(duration), avg(retry_count / duration), countIf(block_count > 0) / count(), countIf(first_time = 0) / count(), uniqHLL12(x_id), uniqHLL12(y_id) group by dt_m, domain)) engine MergeTree partition by toDate(datetime) order by (toStartOfTenMinutes(datetime), domain) settings index_granularity_bytes = 10000000; insert into projection_test with rowNumberInAllBlocks() as id select 1, toDateTime('2020-10-24 00:00:00') + (id / 20), toString(id % 100), * from generateRandom('x_id String, y_id String, block_count Int64, retry_count Int64, duration Int64, kbytes Int64, buffer_time Int64, first_time Int64, total_bytes Nullable(UInt64), valid_bytes Nullable(UInt64), completed_bytes Nullable(UInt64), fixed_bytes Nullable(UInt64), force_bytes Nullable(UInt64)', 10, 10, 1) limit 1000 settings max_threads = 1; diff --git a/tests/queries/0_stateless/01771_bloom_filter_not_has.sql b/tests/queries/0_stateless/01771_bloom_filter_not_has.sql index ab0e3d308f9..f945cbde56b 100644 --- a/tests/queries/0_stateless/01771_bloom_filter_not_has.sql +++ b/tests/queries/0_stateless/01771_bloom_filter_not_has.sql @@ -1,3 +1,4 @@ +-- Tags: no-parallel, long DROP TABLE IF EXISTS bloom_filter_null_array; CREATE TABLE bloom_filter_null_array (v Array(Int32), INDEX idx v TYPE bloom_filter GRANULARITY 3) ENGINE = MergeTree() ORDER BY v; INSERT INTO bloom_filter_null_array SELECT [number] FROM numbers(10000000); diff --git a/tests/queries/0_stateless/01780_column_sparse_distinct.reference b/tests/queries/0_stateless/01780_column_sparse_distinct.reference index bb0cebc6540..beb45208e7b 100644 --- a/tests/queries/0_stateless/01780_column_sparse_distinct.reference +++ b/tests/queries/0_stateless/01780_column_sparse_distinct.reference @@ -5,3 +5,4 @@ all_2_2_0 v Sparse 2 3 4 +5 diff --git a/tests/queries/0_stateless/01780_column_sparse_distinct.sql b/tests/queries/0_stateless/01780_column_sparse_distinct.sql index 502ca7600d4..e98bada1aac 100644 --- a/tests/queries/0_stateless/01780_column_sparse_distinct.sql +++ b/tests/queries/0_stateless/01780_column_sparse_distinct.sql @@ -7,7 +7,7 @@ SETTINGS ratio_of_defaults_for_sparse_serialization = 0.9; SYSTEM STOP MERGES t_sparse_distinct; -INSERT INTO t_sparse_distinct SELECT number, number % 5 FROM numbers(100000); +INSERT INTO t_sparse_distinct SELECT number, number % 6 FROM numbers(100000); INSERT INTO t_sparse_distinct SELECT number, number % 100 = 0 FROM numbers(100000); SELECT name, column, serialization_kind diff --git a/tests/queries/0_stateless/01791_dist_INSERT_block_structure_mismatch.reference b/tests/queries/0_stateless/01791_dist_INSERT_block_structure_mismatch.reference index f3be69d3279..b0d8284faa5 100644 --- a/tests/queries/0_stateless/01791_dist_INSERT_block_structure_mismatch.reference +++ b/tests/queries/0_stateless/01791_dist_INSERT_block_structure_mismatch.reference @@ -1,7 +1,7 @@ DistributedSink: Structure does not match (remote: n Int8 Int8(size = 0), local: n UInt64 UInt64(size = 1)), implicit conversion will be done. DistributedSink: Structure does not match (remote: n Int8 Int8(size = 0), local: n UInt64 UInt64(size = 1)), implicit conversion will be done. - default.dist_01683.DirectoryMonitor: Structure does not match (remote: n Int8 Int8(size = 0), local: n UInt64 UInt64(size = 0)), implicit conversion will be done - default.dist_01683.DirectoryMonitor: Structure does not match (remote: n Int8 Int8(size = 0), local: n UInt64 UInt64(size = 0)), implicit conversion will be done + default.dist_01683.DirectoryMonitor.default: Structure does not match (remote: n Int8 Int8(size = 0), local: n UInt64 UInt64(size = 0)), implicit conversion will be done + default.dist_01683.DirectoryMonitor.default: Structure does not match (remote: n Int8 Int8(size = 0), local: n UInt64 UInt64(size = 0)), implicit conversion will be done 1 1 2 diff --git a/tests/queries/0_stateless/01823_explain_json.reference b/tests/queries/0_stateless/01823_explain_json.reference index 9df7c16e4f4..befbf82f4fb 100644 --- a/tests/queries/0_stateless/01823_explain_json.reference +++ b/tests/queries/0_stateless/01823_explain_json.reference @@ -37,63 +37,59 @@ "Node Type": "Aggregating", "Header": [ { - "Name": "number", + "Name": "number_0", "Type": "UInt64" }, { - "Name": "plus(number, 1)", - "Type": "UInt64" - }, - { - "Name": "quantile(0.2)(number)", + "Name": "quantile(0.2_Float64)(number_0)", "Type": "Float64" }, { - "Name": "sumIf(number, greater(number, 0))", + "Name": "sumIf(number_0, greater(number_0, 0_UInt8))", "Type": "UInt64" } ], - "Keys": ["number", "plus(number, 1)"], + "Keys": ["number_0"], "Aggregates": [ { - "Name": "quantile(0.2)(number)", + "Name": "quantile(0.2_Float64)(number_0)", "Function": { "Name": "quantile", "Parameters": ["0.2"], "Argument Types": ["UInt64"], "Result Type": "Float64" }, - "Arguments": ["number"] + "Arguments": ["number_0"] }, { - "Name": "sumIf(number, greater(number, 0))", + "Name": "sumIf(number_0, greater(number_0, 0_UInt8))", "Function": { "Name": "sumIf", "Argument Types": ["UInt64", "UInt8"], "Result Type": "UInt64" }, - "Arguments": ["number", "greater(number, 0)"] + "Arguments": ["number_0", "greater(number_0, 0_UInt8)"] } ], -------- "Node Type": "ArrayJoin", "Left": false, - "Columns": ["x", "y"], + "Columns": ["x_0", "y_1"], -------- "Node Type": "Distinct", - "Columns": ["intDiv(number, 3)", "intDiv(number, 2)"], + "Columns": ["intDiv(number_0, 2_UInt8)", "intDiv(number_0, 3_UInt8)"], -- "Node Type": "Distinct", - "Columns": ["intDiv(number, 3)", "intDiv(number, 2)"], + "Columns": ["intDiv(number_0, 2_UInt8)", "intDiv(number_0, 3_UInt8)"], -------- "Sort Description": [ { - "Column": "number", + "Column": "number_0", "Ascending": false, "With Fill": false }, { - "Column": "plus(number, 1)", + "Column": "plus(number_0, 1_UInt8)", "Ascending": true, "With Fill": false } diff --git a/tests/queries/0_stateless/01823_explain_json.sh b/tests/queries/0_stateless/01823_explain_json.sh index 7868bc0cc78..39128773069 100755 --- a/tests/queries/0_stateless/01823_explain_json.sh +++ b/tests/queries/0_stateless/01823_explain_json.sh @@ -5,26 +5,29 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh -$CLICKHOUSE_CLIENT -q "EXPLAIN json = 1, description = 0 SELECT 1 UNION ALL SELECT 2 FORMAT TSVRaw" +opts=( + "--allow_experimental_analyzer=1" +) +$CLICKHOUSE_CLIENT "${opts[@]}" -q "EXPLAIN json = 1, description = 0 SELECT 1 UNION ALL SELECT 2 FORMAT TSVRaw" echo "--------" -$CLICKHOUSE_CLIENT -q "explain json = 1, description = 0, header = 1 select 1, 2 + dummy FORMAT TSVRaw" 2> /dev/null | grep Header -m 1 -A 8 +$CLICKHOUSE_CLIENT "${opts[@]}" -q "explain json = 1, description = 0, header = 1 select 1, 2 + dummy FORMAT TSVRaw" 2> /dev/null | grep Header -m 1 -A 8 echo "--------" -$CLICKHOUSE_CLIENT -q "EXPLAIN json = 1, actions = 1, header = 1, description = 0 +$CLICKHOUSE_CLIENT "${opts[@]}" -q "EXPLAIN json = 1, actions = 1, header = 1, description = 0 SELECT quantile(0.2)(number), sumIf(number, number > 0) from numbers(2) group by number, number + 1 FORMAT TSVRaw - " | grep Aggregating -A 40 + " | grep Aggregating -A 36 echo "--------" -$CLICKHOUSE_CLIENT -q "EXPLAIN json = 1, actions = 1, description = 0 +$CLICKHOUSE_CLIENT "${opts[@]}" -q "EXPLAIN json = 1, actions = 1, description = 0 SELECT x, y from numbers(2) array join [number, 1] as x, [number + 1] as y FORMAT TSVRaw " | grep ArrayJoin -A 2 echo "--------" -$CLICKHOUSE_CLIENT -q "EXPLAIN json = 1, actions = 1, description = 0 +$CLICKHOUSE_CLIENT "${opts[@]}" -q "EXPLAIN json = 1, actions = 1, description = 0 SELECT distinct intDiv(number, 2), intDiv(number, 3) from numbers(10) FORMAT TSVRaw " | grep Distinct -A 1 echo "--------" -$CLICKHOUSE_CLIENT -q "EXPLAIN json = 1, actions = 1, description = 0 +$CLICKHOUSE_CLIENT "${opts[@]}" -q "EXPLAIN json = 1, actions = 1, description = 0 SELECT number + 1 from numbers(10) order by number desc, number + 1 limit 3 FORMAT TSVRaw " | grep "Sort Description" -A 12 diff --git a/tests/queries/0_stateless/01881_join_on_conditions_hash.sql.j2 b/tests/queries/0_stateless/01881_join_on_conditions_hash.sql.j2 index cda2b3e08ca..fafefd72cb8 100644 --- a/tests/queries/0_stateless/01881_join_on_conditions_hash.sql.j2 +++ b/tests/queries/0_stateless/01881_join_on_conditions_hash.sql.j2 @@ -1,5 +1,3 @@ -SET allow_experimental_analyzer = 1; - DROP TABLE IF EXISTS t1; DROP TABLE IF EXISTS t2; DROP TABLE IF EXISTS t2_nullable; @@ -72,8 +70,8 @@ SELECT * FROM t1 INNER ALL JOIN t2 ON t1.id == t2.id AND t1.id; -- { serverError SELECT * FROM t1 INNER ALL JOIN t2 ON t1.id == t2.id AND t2.id; -- { serverError 403 } SELECT * FROM t1 INNER ALL JOIN t2 ON t1.id == t2.id AND t1.id + 2; -- { serverError 403 } SELECT * FROM t1 INNER ALL JOIN t2 ON t1.id == t2.id AND t2.id + 2; -- { serverError 403 } -SELECT * FROM t1 INNER ALL JOIN t2 ON t1.id == t2.id AND t1.key; -- { serverError 43 } -SELECT * FROM t1 INNER ALL JOIN t2 ON t1.id == t2.id AND t2.key; -- { serverError 43 } +SELECT * FROM t1 INNER ALL JOIN t2 ON t1.id == t2.id AND t1.key; -- { serverError 43, 403 } +SELECT * FROM t1 INNER ALL JOIN t2 ON t1.id == t2.id AND t2.key; -- { serverError 43, 403 } SELECT * FROM t1 JOIN t2 ON t2.key == t2.key2 AND (t1.id == t2.id OR isNull(t2.key2)); -- { serverError 403 } SELECT * FROM t1 JOIN t2 ON t2.key == t2.key2 OR t1.id == t2.id; -- { serverError 403 } SELECT * FROM t1 JOIN t2 ON (t2.key == t2.key2 AND (t1.key == t1.key2 AND t1.key != 'XXX' OR t1.id == t2.id)) AND t1.id == t2.id; -- { serverError 403 } diff --git a/tests/queries/0_stateless/01881_join_on_conditions_merge.sql.j2 b/tests/queries/0_stateless/01881_join_on_conditions_merge.sql.j2 index c1e9bdb4dc9..e4b704247b2 100644 --- a/tests/queries/0_stateless/01881_join_on_conditions_merge.sql.j2 +++ b/tests/queries/0_stateless/01881_join_on_conditions_merge.sql.j2 @@ -1,5 +1,3 @@ -SET allow_experimental_analyzer = 1; - DROP TABLE IF EXISTS t1; DROP TABLE IF EXISTS t2; DROP TABLE IF EXISTS t2_nullable; @@ -70,8 +68,8 @@ SELECT * FROM t1 INNER ALL JOIN t2 ON t1.id == t2.id AND t1.id; -- { serverError SELECT * FROM t1 INNER ALL JOIN t2 ON t1.id == t2.id AND t2.id; -- { serverError 403 } SELECT * FROM t1 INNER ALL JOIN t2 ON t1.id == t2.id AND t1.id + 2; -- { serverError 403 } SELECT * FROM t1 INNER ALL JOIN t2 ON t1.id == t2.id AND t2.id + 2; -- { serverError 403 } -SELECT * FROM t1 INNER ALL JOIN t2 ON t1.id == t2.id AND t1.key; -- { serverError 43 } -SELECT * FROM t1 INNER ALL JOIN t2 ON t1.id == t2.id AND t2.key; -- { serverError 43 } +SELECT * FROM t1 INNER ALL JOIN t2 ON t1.id == t2.id AND t1.key; -- { serverError 43, 403 } +SELECT * FROM t1 INNER ALL JOIN t2 ON t1.id == t2.id AND t2.key; -- { serverError 43, 403 } SELECT * FROM t1 JOIN t2 ON t2.key == t2.key2 AND (t1.id == t2.id OR isNull(t2.key2)); -- { serverError 403 } SELECT * FROM t1 JOIN t2 ON t2.key == t2.key2 OR t1.id == t2.id; -- { serverError 403 } SELECT * FROM t1 JOIN t2 ON (t2.key == t2.key2 AND (t1.key == t1.key2 AND t1.key != 'XXX' OR t1.id == t2.id)) AND t1.id == t2.id; -- { serverError 403 } diff --git a/tests/queries/0_stateless/01883_with_grouping_sets.reference b/tests/queries/0_stateless/01883_with_grouping_sets.reference index 8fae10a05a4..499e930541f 100644 --- a/tests/queries/0_stateless/01883_with_grouping_sets.reference +++ b/tests/queries/0_stateless/01883_with_grouping_sets.reference @@ -13,8 +13,7 @@ ExpressionTransform Copy 1 → 2 (Expression) ExpressionTransform - (ReadFromStorage) - Memory 0 → 1 + (ReadFromMemoryStorage) 1 0 1 4500 1 0 3 4700 1 0 5 4900 diff --git a/tests/queries/0_stateless/01888_read_int_safe.sql b/tests/queries/0_stateless/01888_read_int_safe.sql index 3aea8e38ab0..197338775c4 100644 --- a/tests/queries/0_stateless/01888_read_int_safe.sql +++ b/tests/queries/0_stateless/01888_read_int_safe.sql @@ -1,10 +1,10 @@ -select toInt64('--1'); -- { serverError 72; } -select toInt64('+-1'); -- { serverError 72; } -select toInt64('++1'); -- { serverError 72; } -select toInt64('++'); -- { serverError 72; } -select toInt64('+'); -- { serverError 72; } -select toInt64('1+1'); -- { serverError 6; } -select toInt64('1-1'); -- { serverError 6; } -select toInt64(''); -- { serverError 32; } +select toInt64('--1'); -- { serverError 72 } +select toInt64('+-1'); -- { serverError 72 } +select toInt64('++1'); -- { serverError 72 } +select toInt64('++'); -- { serverError 72 } +select toInt64('+'); -- { serverError 72 } +select toInt64('1+1'); -- { serverError 6 } +select toInt64('1-1'); -- { serverError 6 } +select toInt64(''); -- { serverError 32 } select toInt64('1'); select toInt64('-1'); diff --git a/tests/queries/0_stateless/01911_logical_error_minus.sql b/tests/queries/0_stateless/01911_logical_error_minus.sql index 3dcdedd38f5..7f371a463f8 100644 --- a/tests/queries/0_stateless/01911_logical_error_minus.sql +++ b/tests/queries/0_stateless/01911_logical_error_minus.sql @@ -26,7 +26,7 @@ INSERT INTO codecTest (key, name, ref_valueF64, valueF64, ref_valueF32, valueF32 INSERT INTO codecTest (key, name, ref_valueF64, valueF64, ref_valueF32, valueF32) SELECT number AS n, 'sin(n*n*n)*n', sin(n * n * n * n* n) AS v, v, v, v FROM system.numbers LIMIT 301, 100; -SELECT IF(-2, NULL, 0.00009999999747378752), IF(1048577, 1048576, NULL), c1.key, IF(1, NULL, NULL), c2.key FROM codecTest AS c1 , codecTest AS c2 WHERE ignore(IF(257, -2, NULL), arrayJoin([65537]), IF(3, 1024, 9223372036854775807)) AND IF(NULL, 256, NULL) AND (IF(NULL, '1048576', NULL) = (c1.key - NULL)) LIMIT 65535; +SELECT IF(2, NULL, 0.00009999999747378752), IF(104, 1048576, NULL), c1.key, IF(1, NULL, NULL), c2.key FROM codecTest AS c1 , codecTest AS c2 WHERE ignore(IF(255, -2, NULL), arrayJoin([65537]), IF(3, 1024, 9223372036854775807)) AND IF(NULL, 256, NULL) AND (IF(NULL, '1048576', NULL) = (c1.key - NULL)) LIMIT 65535; SELECT c1.key, c1.name, c1.ref_valueF64, c1.valueF64, c1.ref_valueF64 - c1.valueF64 AS dF64, '', c2.key, c2.ref_valueF64 FROM codecTest AS c1 , codecTest AS c2 WHERE (dF64 != 3) AND c1.valueF64 != 0 AND (c2.key = (c1.key - 1048576)) LIMIT 0; @@ -72,7 +72,7 @@ INSERT INTO codecTest (key, ref_valueU64, valueU64, ref_valueU32, valueU32, ref_ SELECT number as n, n + (rand64() - 9223372036854775807)/1000 as v, v, v, v, v, v, v, v, v, v, v, v, v, v, v, v, toDateTime(v), toDateTime(v), toDate(v), toDate(v) FROM system.numbers LIMIT 3001, 1000; -SELECT IF(-2, NULL, 0.00009999999747378752), IF(1048577, 1048576, NULL), c1.key, IF(1, NULL, NULL), c2.key FROM codecTest AS c1 , codecTest AS c2 WHERE ignore(IF(257, -2, NULL), arrayJoin([65537]), IF(3, 1024, 9223372036854775807)) AND IF(NULL, 256, NULL) AND (IF(NULL, '1048576', NULL) = (c1.key - NULL)) LIMIT 65535; +SELECT IF(2, NULL, 0.00009999999747378752), IF(104, 1048576, NULL), c1.key, IF(1, NULL, NULL), c2.key FROM codecTest AS c1 , codecTest AS c2 WHERE ignore(IF(255, -2, NULL), arrayJoin([65537]), IF(3, 1024, 9223372036854775807)) AND IF(NULL, 256, NULL) AND (IF(NULL, '1048576', NULL) = (c1.key - NULL)) LIMIT 65535; DROP TABLE codecTest; diff --git a/tests/queries/0_stateless/01913_names_of_tuple_literal.sql b/tests/queries/0_stateless/01913_names_of_tuple_literal.sql index 09de9e8cf37..879f4c91587 100644 --- a/tests/queries/0_stateless/01913_names_of_tuple_literal.sql +++ b/tests/queries/0_stateless/01913_names_of_tuple_literal.sql @@ -1,2 +1,4 @@ +SET allow_experimental_analyzer = 0; + SELECT ((1, 2), (2, 3), (3, 4)) FORMAT TSVWithNames; SELECT ((1, 2), (2, 3), (3, 4)) FORMAT TSVWithNames SETTINGS legacy_column_name_of_tuple_literal = 1; diff --git a/tests/queries/0_stateless/01920_async_drain_connections.sql b/tests/queries/0_stateless/01920_async_drain_connections.sql deleted file mode 100644 index 827ca13fc1a..00000000000 --- a/tests/queries/0_stateless/01920_async_drain_connections.sql +++ /dev/null @@ -1,6 +0,0 @@ -drop table if exists t; - -create table t (number UInt64) engine = Distributed(test_cluster_two_shards, system, numbers); -select * from t where number = 0 limit 2 settings sleep_in_receive_cancel_ms = 10000, max_execution_time = 5; - -drop table t; diff --git a/tests/queries/0_stateless/01934_constexpr_aggregate_function_parameters.sql b/tests/queries/0_stateless/01934_constexpr_aggregate_function_parameters.sql index 730313f1daa..95d411c4cec 100644 --- a/tests/queries/0_stateless/01934_constexpr_aggregate_function_parameters.sql +++ b/tests/queries/0_stateless/01934_constexpr_aggregate_function_parameters.sql @@ -1,12 +1,10 @@ -SET allow_experimental_analyzer = 1; - SELECT groupArray(2 + 3)(number) FROM numbers(10); SELECT groupArray('5'::UInt8)(number) FROM numbers(10); SELECT groupArray(NULL)(number) FROM numbers(10); -- { serverError 36 } SELECT groupArray(NULL + NULL)(number) FROM numbers(10); -- { serverError 36 } SELECT groupArray([])(number) FROM numbers(10); -- { serverError 36 } -SELECT groupArray(throwIf(1))(number) FROM numbers(10); -- { serverError 36 } +SELECT groupArray(throwIf(1))(number) FROM numbers(10); -- { serverError 36, 134 } -- Not the best error message, can be improved. -SELECT groupArray(number)(number) FROM numbers(10); -- { serverError 36 } +SELECT groupArray(number)(number) FROM numbers(10); -- { serverError 36, 47 } diff --git a/tests/queries/0_stateless/02002_global_subqueries_subquery_or_table_name.sql b/tests/queries/0_stateless/02002_global_subqueries_subquery_or_table_name.sql index 75eff67970e..8ac8dc35276 100644 --- a/tests/queries/0_stateless/02002_global_subqueries_subquery_or_table_name.sql +++ b/tests/queries/0_stateless/02002_global_subqueries_subquery_or_table_name.sql @@ -1,9 +1,7 @@ -- Tags: global -SET allow_experimental_analyzer = 1; - SELECT cityHash64(number GLOBAL IN (NULL, -2147483648, -9223372036854775808), nan, 1024, NULL, NULL, 1.000100016593933, NULL), (NULL, cityHash64(inf, -2147483648, NULL, NULL, 10.000100135803223), cityHash64(1.1754943508222875e-38, NULL, NULL, NULL), 2147483647) FROM cluster(test_cluster_two_shards_localhost, numbers((NULL, cityHash64(0., 65536, NULL, NULL, 10000000000., NULL), 0) GLOBAL IN (some_identifier), 65536)) -WHERE number GLOBAL IN [1025] --{serverError 36} +WHERE number GLOBAL IN [1025] --{serverError 36, 284} diff --git a/tests/queries/0_stateless/02008_materialize_column.sql b/tests/queries/0_stateless/02008_materialize_column.sql index 8a8eb2afe83..a78920d2525 100644 --- a/tests/queries/0_stateless/02008_materialize_column.sql +++ b/tests/queries/0_stateless/02008_materialize_column.sql @@ -8,33 +8,33 @@ INSERT INTO tmp SELECT * FROM system.numbers LIMIT 20; ALTER TABLE tmp MATERIALIZE COLUMN x; -- { serverError 36 } ALTER TABLE tmp ADD COLUMN s String DEFAULT toString(x); -SELECT groupArray(x), groupArray(s) FROM tmp; +SELECT arraySort(arraySort(groupArray(x))), groupArray(s) FROM tmp; ALTER TABLE tmp MODIFY COLUMN s String DEFAULT toString(x+1); -SELECT groupArray(x), groupArray(s) FROM tmp; +SELECT arraySort(groupArray(x)), groupArray(s) FROM tmp; ALTER TABLE tmp MATERIALIZE COLUMN s; ALTER TABLE tmp MODIFY COLUMN s String DEFAULT toString(x+2); -SELECT groupArray(x), groupArray(s) FROM tmp; +SELECT arraySort(groupArray(x)), groupArray(s) FROM tmp; ALTER TABLE tmp MATERIALIZE COLUMN s; ALTER TABLE tmp MODIFY COLUMN s String DEFAULT toString(x+3); -SELECT groupArray(x), groupArray(s) FROM tmp; +SELECT arraySort(groupArray(x)), groupArray(s) FROM tmp; ALTER TABLE tmp DROP COLUMN s; ALTER TABLE tmp ADD COLUMN s String MATERIALIZED toString(x); -SELECT groupArray(x), groupArray(s) FROM tmp; +SELECT arraySort(groupArray(x)), groupArray(s) FROM tmp; ALTER TABLE tmp MODIFY COLUMN s String MATERIALIZED toString(x+1); -SELECT groupArray(x), groupArray(s) FROM tmp; +SELECT arraySort(groupArray(x)), groupArray(s) FROM tmp; ALTER TABLE tmp MATERIALIZE COLUMN s; ALTER TABLE tmp MODIFY COLUMN s String MATERIALIZED toString(x+2); -SELECT groupArray(x), groupArray(s) FROM tmp; +SELECT arraySort(groupArray(x)), groupArray(s) FROM tmp; ALTER TABLE tmp MATERIALIZE COLUMN s; ALTER TABLE tmp MODIFY COLUMN s String MATERIALIZED toString(x+3); -SELECT groupArray(x), groupArray(s) FROM tmp; +SELECT arraySort(groupArray(x)), groupArray(s) FROM tmp; ALTER TABLE tmp DROP COLUMN s; DROP TABLE tmp; diff --git a/tests/queries/0_stateless/02008_tuple_to_name_value_pairs.sql b/tests/queries/0_stateless/02008_tuple_to_name_value_pairs.sql index 59987a86590..1f6026bb61e 100644 --- a/tests/queries/0_stateless/02008_tuple_to_name_value_pairs.sql +++ b/tests/queries/0_stateless/02008_tuple_to_name_value_pairs.sql @@ -19,7 +19,7 @@ INSERT INTO test02008 VALUES (tuple(3.3, 5.5, 6.6)); SELECT untuple(arrayJoin(tupleToNameValuePairs(col))) from test02008; DROP TABLE IF EXISTS test02008; -SELECT tupleToNameValuePairs(tuple(1, 1.3)); -- { serverError 43; } -SELECT tupleToNameValuePairs(tuple(1, [1,2])); -- { serverError 43; } -SELECT tupleToNameValuePairs(tuple(1, 'a')); -- { serverError 43; } -SELECT tupleToNameValuePairs(33); -- { serverError 43; } +SELECT tupleToNameValuePairs(tuple(1, 1.3)); -- { serverError 43 } +SELECT tupleToNameValuePairs(tuple(1, [1,2])); -- { serverError 43 } +SELECT tupleToNameValuePairs(tuple(1, 'a')); -- { serverError 43 } +SELECT tupleToNameValuePairs(33); -- { serverError 43 } diff --git a/tests/queries/0_stateless/02015_async_inserts_4.sh b/tests/queries/0_stateless/02015_async_inserts_4.sh index 65598923b96..28f0e250630 100755 --- a/tests/queries/0_stateless/02015_async_inserts_4.sh +++ b/tests/queries/0_stateless/02015_async_inserts_4.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# Tags: no-fasttest CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh diff --git a/tests/queries/0_stateless/02015_async_inserts_7.sh b/tests/queries/0_stateless/02015_async_inserts_7.sh index c8cbbc48a29..29f908cdc90 100755 --- a/tests/queries/0_stateless/02015_async_inserts_7.sh +++ b/tests/queries/0_stateless/02015_async_inserts_7.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# Tags: no-fasttest CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh diff --git a/tests/queries/0_stateless/02048_clickhouse_local_stage.reference b/tests/queries/0_stateless/02048_clickhouse_local_stage.reference index 44c39f2a444..00e0f4ddb2e 100644 --- a/tests/queries/0_stateless/02048_clickhouse_local_stage.reference +++ b/tests/queries/0_stateless/02048_clickhouse_local_stage.reference @@ -1,15 +1,15 @@ -execute: default +execute: --allow_experimental_analyzer=1 "foo" 1 -execute: --stage fetch_columns -"dummy" +execute: --allow_experimental_analyzer=1 --stage fetch_columns +"system.one.dummy_0" 0 -execute: --stage with_mergeable_state -"1" +execute: --allow_experimental_analyzer=1 --stage with_mergeable_state +"1_UInt8" 1 -execute: --stage with_mergeable_state_after_aggregation -"1" +execute: --allow_experimental_analyzer=1 --stage with_mergeable_state_after_aggregation +"1_UInt8" 1 -execute: --stage complete +execute: --allow_experimental_analyzer=1 --stage complete "foo" 1 diff --git a/tests/queries/0_stateless/02048_clickhouse_local_stage.sh b/tests/queries/0_stateless/02048_clickhouse_local_stage.sh index 5c1303b5160..182acc23a13 100755 --- a/tests/queries/0_stateless/02048_clickhouse_local_stage.sh +++ b/tests/queries/0_stateless/02048_clickhouse_local_stage.sh @@ -5,6 +5,10 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh +opts=( + "--allow_experimental_analyzer=1" +) + function execute_query() { if [ $# -eq 0 ]; then @@ -15,8 +19,8 @@ function execute_query() ${CLICKHOUSE_LOCAL} "$@" --format CSVWithNames -q "SELECT 1 AS foo" } -execute_query # default -- complete -execute_query --stage fetch_columns -execute_query --stage with_mergeable_state -execute_query --stage with_mergeable_state_after_aggregation -execute_query --stage complete +execute_query "${opts[@]}" # default -- complete +execute_query "${opts[@]}" --stage fetch_columns +execute_query "${opts[@]}" --stage with_mergeable_state +execute_query "${opts[@]}" --stage with_mergeable_state_after_aggregation +execute_query "${opts[@]}" --stage complete diff --git a/tests/queries/0_stateless/02116_tuple_element.sql b/tests/queries/0_stateless/02116_tuple_element.sql index b43c9133e6b..c911712684d 100644 --- a/tests/queries/0_stateless/02116_tuple_element.sql +++ b/tests/queries/0_stateless/02116_tuple_element.sql @@ -1,5 +1,3 @@ -SET allow_experimental_analyzer = 1; - DROP TABLE IF EXISTS t_tuple_element; CREATE TABLE t_tuple_element(t1 Tuple(a UInt32, s String), t2 Tuple(UInt32, String)) ENGINE = Memory; @@ -18,7 +16,7 @@ EXPLAIN SYNTAX SELECT tupleElement(t1, 'a') FROM t_tuple_element; SELECT tupleElement(number, 1) FROM numbers(1); -- { serverError 43 } SELECT tupleElement(t1) FROM t_tuple_element; -- { serverError 42 } -SELECT tupleElement(t1, 'b') FROM t_tuple_element; -- { serverError 10 } +SELECT tupleElement(t1, 'b') FROM t_tuple_element; -- { serverError 10, 47 } SELECT tupleElement(t1, 0) FROM t_tuple_element; -- { serverError 127 } SELECT tupleElement(t1, 3) FROM t_tuple_element; -- { serverError 127 } SELECT tupleElement(t1, materialize('a')) FROM t_tuple_element; -- { serverError 43 } @@ -30,7 +28,7 @@ SELECT tupleElement(t2, 1) FROM t_tuple_element; EXPLAIN SYNTAX SELECT tupleElement(t2, 1) FROM t_tuple_element; SELECT tupleElement(t2) FROM t_tuple_element; -- { serverError 42 } -SELECT tupleElement(t2, 'a') FROM t_tuple_element; -- { serverError 10 } +SELECT tupleElement(t2, 'a') FROM t_tuple_element; -- { serverError 10, 47 } SELECT tupleElement(t2, 0) FROM t_tuple_element; -- { serverError 127 } SELECT tupleElement(t2, 3) FROM t_tuple_element; -- { serverError 127 } SELECT tupleElement(t2, materialize(1)) FROM t_tuple_element; -- { serverError 43 } diff --git a/tests/queries/0_stateless/02117_show_create_table_system.reference b/tests/queries/0_stateless/02117_show_create_table_system.reference index f77076bcd5c..c13de3faec3 100644 --- a/tests/queries/0_stateless/02117_show_create_table_system.reference +++ b/tests/queries/0_stateless/02117_show_create_table_system.reference @@ -289,7 +289,7 @@ CREATE TABLE system.grants ( `user_name` Nullable(String), `role_name` Nullable(String), - `access_type` Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW FILESYSTEM CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER NAMED COLLECTION' = 41, 'ALTER TABLE' = 42, 'ALTER DATABASE' = 43, 'ALTER VIEW REFRESH' = 44, 'ALTER VIEW MODIFY QUERY' = 45, 'ALTER VIEW' = 46, 'ALTER' = 47, 'CREATE DATABASE' = 48, 'CREATE TABLE' = 49, 'CREATE VIEW' = 50, 'CREATE DICTIONARY' = 51, 'CREATE TEMPORARY TABLE' = 52, 'CREATE FUNCTION' = 53, 'CREATE NAMED COLLECTION' = 54, 'CREATE' = 55, 'DROP DATABASE' = 56, 'DROP TABLE' = 57, 'DROP VIEW' = 58, 'DROP DICTIONARY' = 59, 'DROP FUNCTION' = 60, 'DROP NAMED COLLECTION' = 61, 'DROP' = 62, 'TRUNCATE' = 63, 'OPTIMIZE' = 64, 'BACKUP' = 65, 'KILL QUERY' = 66, 'KILL TRANSACTION' = 67, 'MOVE PARTITION BETWEEN SHARDS' = 68, 'CREATE USER' = 69, 'ALTER USER' = 70, 'DROP USER' = 71, 'CREATE ROLE' = 72, 'ALTER ROLE' = 73, 'DROP ROLE' = 74, 'ROLE ADMIN' = 75, 'CREATE ROW POLICY' = 76, 'ALTER ROW POLICY' = 77, 'DROP ROW POLICY' = 78, 'CREATE QUOTA' = 79, 'ALTER QUOTA' = 80, 'DROP QUOTA' = 81, 'CREATE SETTINGS PROFILE' = 82, 'ALTER SETTINGS PROFILE' = 83, 'DROP SETTINGS PROFILE' = 84, 'SHOW USERS' = 85, 'SHOW ROLES' = 86, 'SHOW ROW POLICIES' = 87, 'SHOW QUOTAS' = 88, 'SHOW SETTINGS PROFILES' = 89, 'SHOW ACCESS' = 90, 'SHOW NAMED COLLECTIONS' = 91, 'SHOW NAMED COLLECTIONS SECRETS' = 92, 'ACCESS MANAGEMENT' = 93, 'SYSTEM SHUTDOWN' = 94, 'SYSTEM DROP DNS CACHE' = 95, 'SYSTEM DROP MARK CACHE' = 96, 'SYSTEM DROP UNCOMPRESSED CACHE' = 97, 'SYSTEM DROP MMAP CACHE' = 98, 'SYSTEM DROP QUERY CACHE' = 99, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 100, 'SYSTEM DROP FILESYSTEM CACHE' = 101, 'SYSTEM DROP SCHEMA CACHE' = 102, 'SYSTEM DROP S3 CLIENT CACHE' = 103, 'SYSTEM DROP CACHE' = 104, 'SYSTEM RELOAD CONFIG' = 105, 'SYSTEM RELOAD USERS' = 106, 'SYSTEM RELOAD SYMBOLS' = 107, 'SYSTEM RELOAD DICTIONARY' = 108, 'SYSTEM RELOAD MODEL' = 109, 'SYSTEM RELOAD FUNCTION' = 110, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 111, 'SYSTEM RELOAD' = 112, 'SYSTEM RESTART DISK' = 113, 'SYSTEM MERGES' = 114, 'SYSTEM TTL MERGES' = 115, 'SYSTEM FETCHES' = 116, 'SYSTEM MOVES' = 117, 'SYSTEM DISTRIBUTED SENDS' = 118, 'SYSTEM REPLICATED SENDS' = 119, 'SYSTEM SENDS' = 120, 'SYSTEM REPLICATION QUEUES' = 121, 'SYSTEM DROP REPLICA' = 122, 'SYSTEM SYNC REPLICA' = 123, 'SYSTEM RESTART REPLICA' = 124, 'SYSTEM RESTORE REPLICA' = 125, 'SYSTEM WAIT LOADING PARTS' = 126, 'SYSTEM SYNC DATABASE REPLICA' = 127, 'SYSTEM SYNC TRANSACTION LOG' = 128, 'SYSTEM SYNC FILE CACHE' = 129, 'SYSTEM FLUSH DISTRIBUTED' = 130, 'SYSTEM FLUSH LOGS' = 131, 'SYSTEM FLUSH' = 132, 'SYSTEM THREAD FUZZER' = 133, 'SYSTEM UNFREEZE' = 134, 'SYSTEM' = 135, 'dictGet' = 136, 'addressToLine' = 137, 'addressToLineWithInlines' = 138, 'addressToSymbol' = 139, 'demangle' = 140, 'INTROSPECTION' = 141, 'FILE' = 142, 'URL' = 143, 'REMOTE' = 144, 'MONGO' = 145, 'MEILISEARCH' = 146, 'MYSQL' = 147, 'POSTGRES' = 148, 'SQLITE' = 149, 'ODBC' = 150, 'JDBC' = 151, 'HDFS' = 152, 'S3' = 153, 'HIVE' = 154, 'SOURCES' = 155, 'CLUSTER' = 156, 'ALL' = 157, 'NONE' = 158), + `access_type` Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW FILESYSTEM CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER NAMED COLLECTION' = 41, 'ALTER TABLE' = 42, 'ALTER DATABASE' = 43, 'ALTER VIEW REFRESH' = 44, 'ALTER VIEW MODIFY QUERY' = 45, 'ALTER VIEW' = 46, 'ALTER' = 47, 'CREATE DATABASE' = 48, 'CREATE TABLE' = 49, 'CREATE VIEW' = 50, 'CREATE DICTIONARY' = 51, 'CREATE TEMPORARY TABLE' = 52, 'CREATE ARBITRARY TEMPORARY TABLE' = 53, 'CREATE FUNCTION' = 54, 'CREATE NAMED COLLECTION' = 55, 'CREATE' = 56, 'DROP DATABASE' = 57, 'DROP TABLE' = 58, 'DROP VIEW' = 59, 'DROP DICTIONARY' = 60, 'DROP FUNCTION' = 61, 'DROP NAMED COLLECTION' = 62, 'DROP' = 63, 'TRUNCATE' = 64, 'OPTIMIZE' = 65, 'BACKUP' = 66, 'KILL QUERY' = 67, 'KILL TRANSACTION' = 68, 'MOVE PARTITION BETWEEN SHARDS' = 69, 'CREATE USER' = 70, 'ALTER USER' = 71, 'DROP USER' = 72, 'CREATE ROLE' = 73, 'ALTER ROLE' = 74, 'DROP ROLE' = 75, 'ROLE ADMIN' = 76, 'CREATE ROW POLICY' = 77, 'ALTER ROW POLICY' = 78, 'DROP ROW POLICY' = 79, 'CREATE QUOTA' = 80, 'ALTER QUOTA' = 81, 'DROP QUOTA' = 82, 'CREATE SETTINGS PROFILE' = 83, 'ALTER SETTINGS PROFILE' = 84, 'DROP SETTINGS PROFILE' = 85, 'SHOW USERS' = 86, 'SHOW ROLES' = 87, 'SHOW ROW POLICIES' = 88, 'SHOW QUOTAS' = 89, 'SHOW SETTINGS PROFILES' = 90, 'SHOW ACCESS' = 91, 'SHOW NAMED COLLECTIONS' = 92, 'SHOW NAMED COLLECTIONS SECRETS' = 93, 'ACCESS MANAGEMENT' = 94, 'SYSTEM SHUTDOWN' = 95, 'SYSTEM DROP DNS CACHE' = 96, 'SYSTEM DROP MARK CACHE' = 97, 'SYSTEM DROP UNCOMPRESSED CACHE' = 98, 'SYSTEM DROP MMAP CACHE' = 99, 'SYSTEM DROP QUERY CACHE' = 100, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 101, 'SYSTEM DROP FILESYSTEM CACHE' = 102, 'SYSTEM DROP SCHEMA CACHE' = 103, 'SYSTEM DROP S3 CLIENT CACHE' = 104, 'SYSTEM DROP CACHE' = 105, 'SYSTEM RELOAD CONFIG' = 106, 'SYSTEM RELOAD USERS' = 107, 'SYSTEM RELOAD SYMBOLS' = 108, 'SYSTEM RELOAD DICTIONARY' = 109, 'SYSTEM RELOAD MODEL' = 110, 'SYSTEM RELOAD FUNCTION' = 111, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 112, 'SYSTEM RELOAD' = 113, 'SYSTEM RESTART DISK' = 114, 'SYSTEM MERGES' = 115, 'SYSTEM TTL MERGES' = 116, 'SYSTEM FETCHES' = 117, 'SYSTEM MOVES' = 118, 'SYSTEM DISTRIBUTED SENDS' = 119, 'SYSTEM REPLICATED SENDS' = 120, 'SYSTEM SENDS' = 121, 'SYSTEM REPLICATION QUEUES' = 122, 'SYSTEM DROP REPLICA' = 123, 'SYSTEM SYNC REPLICA' = 124, 'SYSTEM RESTART REPLICA' = 125, 'SYSTEM RESTORE REPLICA' = 126, 'SYSTEM WAIT LOADING PARTS' = 127, 'SYSTEM SYNC DATABASE REPLICA' = 128, 'SYSTEM SYNC TRANSACTION LOG' = 129, 'SYSTEM SYNC FILE CACHE' = 130, 'SYSTEM FLUSH DISTRIBUTED' = 131, 'SYSTEM FLUSH LOGS' = 132, 'SYSTEM FLUSH' = 133, 'SYSTEM THREAD FUZZER' = 134, 'SYSTEM UNFREEZE' = 135, 'SYSTEM' = 136, 'dictGet' = 137, 'addressToLine' = 138, 'addressToLineWithInlines' = 139, 'addressToSymbol' = 140, 'demangle' = 141, 'INTROSPECTION' = 142, 'FILE' = 143, 'URL' = 144, 'REMOTE' = 145, 'MONGO' = 146, 'MEILISEARCH' = 147, 'MYSQL' = 148, 'POSTGRES' = 149, 'SQLITE' = 150, 'ODBC' = 151, 'JDBC' = 152, 'HDFS' = 153, 'S3' = 154, 'HIVE' = 155, 'SOURCES' = 156, 'CLUSTER' = 157, 'ALL' = 158, 'NONE' = 159), `database` Nullable(String), `table` Nullable(String), `column` Nullable(String), @@ -570,10 +570,10 @@ ENGINE = SystemPartsColumns COMMENT 'SYSTEM TABLE is built on the fly.' CREATE TABLE system.privileges ( - `privilege` Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW FILESYSTEM CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER NAMED COLLECTION' = 41, 'ALTER TABLE' = 42, 'ALTER DATABASE' = 43, 'ALTER VIEW REFRESH' = 44, 'ALTER VIEW MODIFY QUERY' = 45, 'ALTER VIEW' = 46, 'ALTER' = 47, 'CREATE DATABASE' = 48, 'CREATE TABLE' = 49, 'CREATE VIEW' = 50, 'CREATE DICTIONARY' = 51, 'CREATE TEMPORARY TABLE' = 52, 'CREATE FUNCTION' = 53, 'CREATE NAMED COLLECTION' = 54, 'CREATE' = 55, 'DROP DATABASE' = 56, 'DROP TABLE' = 57, 'DROP VIEW' = 58, 'DROP DICTIONARY' = 59, 'DROP FUNCTION' = 60, 'DROP NAMED COLLECTION' = 61, 'DROP' = 62, 'TRUNCATE' = 63, 'OPTIMIZE' = 64, 'BACKUP' = 65, 'KILL QUERY' = 66, 'KILL TRANSACTION' = 67, 'MOVE PARTITION BETWEEN SHARDS' = 68, 'CREATE USER' = 69, 'ALTER USER' = 70, 'DROP USER' = 71, 'CREATE ROLE' = 72, 'ALTER ROLE' = 73, 'DROP ROLE' = 74, 'ROLE ADMIN' = 75, 'CREATE ROW POLICY' = 76, 'ALTER ROW POLICY' = 77, 'DROP ROW POLICY' = 78, 'CREATE QUOTA' = 79, 'ALTER QUOTA' = 80, 'DROP QUOTA' = 81, 'CREATE SETTINGS PROFILE' = 82, 'ALTER SETTINGS PROFILE' = 83, 'DROP SETTINGS PROFILE' = 84, 'SHOW USERS' = 85, 'SHOW ROLES' = 86, 'SHOW ROW POLICIES' = 87, 'SHOW QUOTAS' = 88, 'SHOW SETTINGS PROFILES' = 89, 'SHOW ACCESS' = 90, 'SHOW NAMED COLLECTIONS' = 91, 'SHOW NAMED COLLECTIONS SECRETS' = 92, 'ACCESS MANAGEMENT' = 93, 'SYSTEM SHUTDOWN' = 94, 'SYSTEM DROP DNS CACHE' = 95, 'SYSTEM DROP MARK CACHE' = 96, 'SYSTEM DROP UNCOMPRESSED CACHE' = 97, 'SYSTEM DROP MMAP CACHE' = 98, 'SYSTEM DROP QUERY CACHE' = 99, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 100, 'SYSTEM DROP FILESYSTEM CACHE' = 101, 'SYSTEM DROP SCHEMA CACHE' = 102, 'SYSTEM DROP S3 CLIENT CACHE' = 103, 'SYSTEM DROP CACHE' = 104, 'SYSTEM RELOAD CONFIG' = 105, 'SYSTEM RELOAD USERS' = 106, 'SYSTEM RELOAD SYMBOLS' = 107, 'SYSTEM RELOAD DICTIONARY' = 108, 'SYSTEM RELOAD MODEL' = 109, 'SYSTEM RELOAD FUNCTION' = 110, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 111, 'SYSTEM RELOAD' = 112, 'SYSTEM RESTART DISK' = 113, 'SYSTEM MERGES' = 114, 'SYSTEM TTL MERGES' = 115, 'SYSTEM FETCHES' = 116, 'SYSTEM MOVES' = 117, 'SYSTEM DISTRIBUTED SENDS' = 118, 'SYSTEM REPLICATED SENDS' = 119, 'SYSTEM SENDS' = 120, 'SYSTEM REPLICATION QUEUES' = 121, 'SYSTEM DROP REPLICA' = 122, 'SYSTEM SYNC REPLICA' = 123, 'SYSTEM RESTART REPLICA' = 124, 'SYSTEM RESTORE REPLICA' = 125, 'SYSTEM WAIT LOADING PARTS' = 126, 'SYSTEM SYNC DATABASE REPLICA' = 127, 'SYSTEM SYNC TRANSACTION LOG' = 128, 'SYSTEM SYNC FILE CACHE' = 129, 'SYSTEM FLUSH DISTRIBUTED' = 130, 'SYSTEM FLUSH LOGS' = 131, 'SYSTEM FLUSH' = 132, 'SYSTEM THREAD FUZZER' = 133, 'SYSTEM UNFREEZE' = 134, 'SYSTEM' = 135, 'dictGet' = 136, 'addressToLine' = 137, 'addressToLineWithInlines' = 138, 'addressToSymbol' = 139, 'demangle' = 140, 'INTROSPECTION' = 141, 'FILE' = 142, 'URL' = 143, 'REMOTE' = 144, 'MONGO' = 145, 'MEILISEARCH' = 146, 'MYSQL' = 147, 'POSTGRES' = 148, 'SQLITE' = 149, 'ODBC' = 150, 'JDBC' = 151, 'HDFS' = 152, 'S3' = 153, 'HIVE' = 154, 'SOURCES' = 155, 'CLUSTER' = 156, 'ALL' = 157, 'NONE' = 158), + `privilege` Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW FILESYSTEM CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER NAMED COLLECTION' = 41, 'ALTER TABLE' = 42, 'ALTER DATABASE' = 43, 'ALTER VIEW REFRESH' = 44, 'ALTER VIEW MODIFY QUERY' = 45, 'ALTER VIEW' = 46, 'ALTER' = 47, 'CREATE DATABASE' = 48, 'CREATE TABLE' = 49, 'CREATE VIEW' = 50, 'CREATE DICTIONARY' = 51, 'CREATE TEMPORARY TABLE' = 52, 'CREATE ARBITRARY TEMPORARY TABLE' = 53, 'CREATE FUNCTION' = 54, 'CREATE NAMED COLLECTION' = 55, 'CREATE' = 56, 'DROP DATABASE' = 57, 'DROP TABLE' = 58, 'DROP VIEW' = 59, 'DROP DICTIONARY' = 60, 'DROP FUNCTION' = 61, 'DROP NAMED COLLECTION' = 62, 'DROP' = 63, 'TRUNCATE' = 64, 'OPTIMIZE' = 65, 'BACKUP' = 66, 'KILL QUERY' = 67, 'KILL TRANSACTION' = 68, 'MOVE PARTITION BETWEEN SHARDS' = 69, 'CREATE USER' = 70, 'ALTER USER' = 71, 'DROP USER' = 72, 'CREATE ROLE' = 73, 'ALTER ROLE' = 74, 'DROP ROLE' = 75, 'ROLE ADMIN' = 76, 'CREATE ROW POLICY' = 77, 'ALTER ROW POLICY' = 78, 'DROP ROW POLICY' = 79, 'CREATE QUOTA' = 80, 'ALTER QUOTA' = 81, 'DROP QUOTA' = 82, 'CREATE SETTINGS PROFILE' = 83, 'ALTER SETTINGS PROFILE' = 84, 'DROP SETTINGS PROFILE' = 85, 'SHOW USERS' = 86, 'SHOW ROLES' = 87, 'SHOW ROW POLICIES' = 88, 'SHOW QUOTAS' = 89, 'SHOW SETTINGS PROFILES' = 90, 'SHOW ACCESS' = 91, 'SHOW NAMED COLLECTIONS' = 92, 'SHOW NAMED COLLECTIONS SECRETS' = 93, 'ACCESS MANAGEMENT' = 94, 'SYSTEM SHUTDOWN' = 95, 'SYSTEM DROP DNS CACHE' = 96, 'SYSTEM DROP MARK CACHE' = 97, 'SYSTEM DROP UNCOMPRESSED CACHE' = 98, 'SYSTEM DROP MMAP CACHE' = 99, 'SYSTEM DROP QUERY CACHE' = 100, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 101, 'SYSTEM DROP FILESYSTEM CACHE' = 102, 'SYSTEM DROP SCHEMA CACHE' = 103, 'SYSTEM DROP S3 CLIENT CACHE' = 104, 'SYSTEM DROP CACHE' = 105, 'SYSTEM RELOAD CONFIG' = 106, 'SYSTEM RELOAD USERS' = 107, 'SYSTEM RELOAD SYMBOLS' = 108, 'SYSTEM RELOAD DICTIONARY' = 109, 'SYSTEM RELOAD MODEL' = 110, 'SYSTEM RELOAD FUNCTION' = 111, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 112, 'SYSTEM RELOAD' = 113, 'SYSTEM RESTART DISK' = 114, 'SYSTEM MERGES' = 115, 'SYSTEM TTL MERGES' = 116, 'SYSTEM FETCHES' = 117, 'SYSTEM MOVES' = 118, 'SYSTEM DISTRIBUTED SENDS' = 119, 'SYSTEM REPLICATED SENDS' = 120, 'SYSTEM SENDS' = 121, 'SYSTEM REPLICATION QUEUES' = 122, 'SYSTEM DROP REPLICA' = 123, 'SYSTEM SYNC REPLICA' = 124, 'SYSTEM RESTART REPLICA' = 125, 'SYSTEM RESTORE REPLICA' = 126, 'SYSTEM WAIT LOADING PARTS' = 127, 'SYSTEM SYNC DATABASE REPLICA' = 128, 'SYSTEM SYNC TRANSACTION LOG' = 129, 'SYSTEM SYNC FILE CACHE' = 130, 'SYSTEM FLUSH DISTRIBUTED' = 131, 'SYSTEM FLUSH LOGS' = 132, 'SYSTEM FLUSH' = 133, 'SYSTEM THREAD FUZZER' = 134, 'SYSTEM UNFREEZE' = 135, 'SYSTEM' = 136, 'dictGet' = 137, 'addressToLine' = 138, 'addressToLineWithInlines' = 139, 'addressToSymbol' = 140, 'demangle' = 141, 'INTROSPECTION' = 142, 'FILE' = 143, 'URL' = 144, 'REMOTE' = 145, 'MONGO' = 146, 'MEILISEARCH' = 147, 'MYSQL' = 148, 'POSTGRES' = 149, 'SQLITE' = 150, 'ODBC' = 151, 'JDBC' = 152, 'HDFS' = 153, 'S3' = 154, 'HIVE' = 155, 'SOURCES' = 156, 'CLUSTER' = 157, 'ALL' = 158, 'NONE' = 159), `aliases` Array(String), `level` Nullable(Enum8('GLOBAL' = 0, 'DATABASE' = 1, 'TABLE' = 2, 'DICTIONARY' = 3, 'VIEW' = 4, 'COLUMN' = 5)), - `parent_group` Nullable(Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW FILESYSTEM CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER NAMED COLLECTION' = 41, 'ALTER TABLE' = 42, 'ALTER DATABASE' = 43, 'ALTER VIEW REFRESH' = 44, 'ALTER VIEW MODIFY QUERY' = 45, 'ALTER VIEW' = 46, 'ALTER' = 47, 'CREATE DATABASE' = 48, 'CREATE TABLE' = 49, 'CREATE VIEW' = 50, 'CREATE DICTIONARY' = 51, 'CREATE TEMPORARY TABLE' = 52, 'CREATE FUNCTION' = 53, 'CREATE NAMED COLLECTION' = 54, 'CREATE' = 55, 'DROP DATABASE' = 56, 'DROP TABLE' = 57, 'DROP VIEW' = 58, 'DROP DICTIONARY' = 59, 'DROP FUNCTION' = 60, 'DROP NAMED COLLECTION' = 61, 'DROP' = 62, 'TRUNCATE' = 63, 'OPTIMIZE' = 64, 'BACKUP' = 65, 'KILL QUERY' = 66, 'KILL TRANSACTION' = 67, 'MOVE PARTITION BETWEEN SHARDS' = 68, 'CREATE USER' = 69, 'ALTER USER' = 70, 'DROP USER' = 71, 'CREATE ROLE' = 72, 'ALTER ROLE' = 73, 'DROP ROLE' = 74, 'ROLE ADMIN' = 75, 'CREATE ROW POLICY' = 76, 'ALTER ROW POLICY' = 77, 'DROP ROW POLICY' = 78, 'CREATE QUOTA' = 79, 'ALTER QUOTA' = 80, 'DROP QUOTA' = 81, 'CREATE SETTINGS PROFILE' = 82, 'ALTER SETTINGS PROFILE' = 83, 'DROP SETTINGS PROFILE' = 84, 'SHOW USERS' = 85, 'SHOW ROLES' = 86, 'SHOW ROW POLICIES' = 87, 'SHOW QUOTAS' = 88, 'SHOW SETTINGS PROFILES' = 89, 'SHOW ACCESS' = 90, 'SHOW NAMED COLLECTIONS' = 91, 'SHOW NAMED COLLECTIONS SECRETS' = 92, 'ACCESS MANAGEMENT' = 93, 'SYSTEM SHUTDOWN' = 94, 'SYSTEM DROP DNS CACHE' = 95, 'SYSTEM DROP MARK CACHE' = 96, 'SYSTEM DROP UNCOMPRESSED CACHE' = 97, 'SYSTEM DROP MMAP CACHE' = 98, 'SYSTEM DROP QUERY CACHE' = 99, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 100, 'SYSTEM DROP FILESYSTEM CACHE' = 101, 'SYSTEM DROP SCHEMA CACHE' = 102, 'SYSTEM DROP S3 CLIENT CACHE' = 103, 'SYSTEM DROP CACHE' = 104, 'SYSTEM RELOAD CONFIG' = 105, 'SYSTEM RELOAD USERS' = 106, 'SYSTEM RELOAD SYMBOLS' = 107, 'SYSTEM RELOAD DICTIONARY' = 108, 'SYSTEM RELOAD MODEL' = 109, 'SYSTEM RELOAD FUNCTION' = 110, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 111, 'SYSTEM RELOAD' = 112, 'SYSTEM RESTART DISK' = 113, 'SYSTEM MERGES' = 114, 'SYSTEM TTL MERGES' = 115, 'SYSTEM FETCHES' = 116, 'SYSTEM MOVES' = 117, 'SYSTEM DISTRIBUTED SENDS' = 118, 'SYSTEM REPLICATED SENDS' = 119, 'SYSTEM SENDS' = 120, 'SYSTEM REPLICATION QUEUES' = 121, 'SYSTEM DROP REPLICA' = 122, 'SYSTEM SYNC REPLICA' = 123, 'SYSTEM RESTART REPLICA' = 124, 'SYSTEM RESTORE REPLICA' = 125, 'SYSTEM WAIT LOADING PARTS' = 126, 'SYSTEM SYNC DATABASE REPLICA' = 127, 'SYSTEM SYNC TRANSACTION LOG' = 128, 'SYSTEM SYNC FILE CACHE' = 129, 'SYSTEM FLUSH DISTRIBUTED' = 130, 'SYSTEM FLUSH LOGS' = 131, 'SYSTEM FLUSH' = 132, 'SYSTEM THREAD FUZZER' = 133, 'SYSTEM UNFREEZE' = 134, 'SYSTEM' = 135, 'dictGet' = 136, 'addressToLine' = 137, 'addressToLineWithInlines' = 138, 'addressToSymbol' = 139, 'demangle' = 140, 'INTROSPECTION' = 141, 'FILE' = 142, 'URL' = 143, 'REMOTE' = 144, 'MONGO' = 145, 'MEILISEARCH' = 146, 'MYSQL' = 147, 'POSTGRES' = 148, 'SQLITE' = 149, 'ODBC' = 150, 'JDBC' = 151, 'HDFS' = 152, 'S3' = 153, 'HIVE' = 154, 'SOURCES' = 155, 'CLUSTER' = 156, 'ALL' = 157, 'NONE' = 158)) + `parent_group` Nullable(Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW FILESYSTEM CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER NAMED COLLECTION' = 41, 'ALTER TABLE' = 42, 'ALTER DATABASE' = 43, 'ALTER VIEW REFRESH' = 44, 'ALTER VIEW MODIFY QUERY' = 45, 'ALTER VIEW' = 46, 'ALTER' = 47, 'CREATE DATABASE' = 48, 'CREATE TABLE' = 49, 'CREATE VIEW' = 50, 'CREATE DICTIONARY' = 51, 'CREATE TEMPORARY TABLE' = 52, 'CREATE ARBITRARY TEMPORARY TABLE' = 53, 'CREATE FUNCTION' = 54, 'CREATE NAMED COLLECTION' = 55, 'CREATE' = 56, 'DROP DATABASE' = 57, 'DROP TABLE' = 58, 'DROP VIEW' = 59, 'DROP DICTIONARY' = 60, 'DROP FUNCTION' = 61, 'DROP NAMED COLLECTION' = 62, 'DROP' = 63, 'TRUNCATE' = 64, 'OPTIMIZE' = 65, 'BACKUP' = 66, 'KILL QUERY' = 67, 'KILL TRANSACTION' = 68, 'MOVE PARTITION BETWEEN SHARDS' = 69, 'CREATE USER' = 70, 'ALTER USER' = 71, 'DROP USER' = 72, 'CREATE ROLE' = 73, 'ALTER ROLE' = 74, 'DROP ROLE' = 75, 'ROLE ADMIN' = 76, 'CREATE ROW POLICY' = 77, 'ALTER ROW POLICY' = 78, 'DROP ROW POLICY' = 79, 'CREATE QUOTA' = 80, 'ALTER QUOTA' = 81, 'DROP QUOTA' = 82, 'CREATE SETTINGS PROFILE' = 83, 'ALTER SETTINGS PROFILE' = 84, 'DROP SETTINGS PROFILE' = 85, 'SHOW USERS' = 86, 'SHOW ROLES' = 87, 'SHOW ROW POLICIES' = 88, 'SHOW QUOTAS' = 89, 'SHOW SETTINGS PROFILES' = 90, 'SHOW ACCESS' = 91, 'SHOW NAMED COLLECTIONS' = 92, 'SHOW NAMED COLLECTIONS SECRETS' = 93, 'ACCESS MANAGEMENT' = 94, 'SYSTEM SHUTDOWN' = 95, 'SYSTEM DROP DNS CACHE' = 96, 'SYSTEM DROP MARK CACHE' = 97, 'SYSTEM DROP UNCOMPRESSED CACHE' = 98, 'SYSTEM DROP MMAP CACHE' = 99, 'SYSTEM DROP QUERY CACHE' = 100, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 101, 'SYSTEM DROP FILESYSTEM CACHE' = 102, 'SYSTEM DROP SCHEMA CACHE' = 103, 'SYSTEM DROP S3 CLIENT CACHE' = 104, 'SYSTEM DROP CACHE' = 105, 'SYSTEM RELOAD CONFIG' = 106, 'SYSTEM RELOAD USERS' = 107, 'SYSTEM RELOAD SYMBOLS' = 108, 'SYSTEM RELOAD DICTIONARY' = 109, 'SYSTEM RELOAD MODEL' = 110, 'SYSTEM RELOAD FUNCTION' = 111, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 112, 'SYSTEM RELOAD' = 113, 'SYSTEM RESTART DISK' = 114, 'SYSTEM MERGES' = 115, 'SYSTEM TTL MERGES' = 116, 'SYSTEM FETCHES' = 117, 'SYSTEM MOVES' = 118, 'SYSTEM DISTRIBUTED SENDS' = 119, 'SYSTEM REPLICATED SENDS' = 120, 'SYSTEM SENDS' = 121, 'SYSTEM REPLICATION QUEUES' = 122, 'SYSTEM DROP REPLICA' = 123, 'SYSTEM SYNC REPLICA' = 124, 'SYSTEM RESTART REPLICA' = 125, 'SYSTEM RESTORE REPLICA' = 126, 'SYSTEM WAIT LOADING PARTS' = 127, 'SYSTEM SYNC DATABASE REPLICA' = 128, 'SYSTEM SYNC TRANSACTION LOG' = 129, 'SYSTEM SYNC FILE CACHE' = 130, 'SYSTEM FLUSH DISTRIBUTED' = 131, 'SYSTEM FLUSH LOGS' = 132, 'SYSTEM FLUSH' = 133, 'SYSTEM THREAD FUZZER' = 134, 'SYSTEM UNFREEZE' = 135, 'SYSTEM' = 136, 'dictGet' = 137, 'addressToLine' = 138, 'addressToLineWithInlines' = 139, 'addressToSymbol' = 140, 'demangle' = 141, 'INTROSPECTION' = 142, 'FILE' = 143, 'URL' = 144, 'REMOTE' = 145, 'MONGO' = 146, 'MEILISEARCH' = 147, 'MYSQL' = 148, 'POSTGRES' = 149, 'SQLITE' = 150, 'ODBC' = 151, 'JDBC' = 152, 'HDFS' = 153, 'S3' = 154, 'HIVE' = 155, 'SOURCES' = 156, 'CLUSTER' = 157, 'ALL' = 158, 'NONE' = 159)) ) ENGINE = SystemPrivileges COMMENT 'SYSTEM TABLE is built on the fly.' diff --git a/tests/queries/0_stateless/02125_constant_if_condition_and_not_existing_column.sql b/tests/queries/0_stateless/02125_constant_if_condition_and_not_existing_column.sql index 4aad7ae3694..822ffb19764 100644 --- a/tests/queries/0_stateless/02125_constant_if_condition_and_not_existing_column.sql +++ b/tests/queries/0_stateless/02125_constant_if_condition_and_not_existing_column.sql @@ -6,9 +6,9 @@ insert into test values (0); select if(0, y, 42) from test; select if(1, 42, y) from test; select if(toUInt8(0), y, 42) from test; -select if(toInt8(0), y, 42) from test; +select if(toUInt8(0), y, 42) from test; +select if(toUInt8(1), 42, y) from test; select if(toUInt8(1), 42, y) from test; -select if(toInt8(1), 42, y) from test; select if(toUInt8(toUInt8(0)), y, 42) from test; select if(cast(cast(0, 'UInt8'), 'UInt8'), y, 42) from test; explain syntax select x, if((select hasColumnInTable(currentDatabase(), 'test', 'y')), y, x || '_') from test; diff --git a/tests/queries/0_stateless/02125_query_views_log_window_function.sql b/tests/queries/0_stateless/02125_query_views_log_window_function.sql index 1de2cc95b14..fff1e943c58 100644 --- a/tests/queries/0_stateless/02125_query_views_log_window_function.sql +++ b/tests/queries/0_stateless/02125_query_views_log_window_function.sql @@ -1,4 +1,6 @@ +set allow_experimental_analyzer = 0; set allow_experimental_window_view = 1; + CREATE TABLE data ( `id` UInt64, `timestamp` DateTime) ENGINE = Memory; CREATE WINDOW VIEW wv Engine Memory as select count(id), tumbleStart(w_id) as window_start from data group by tumble(timestamp, INTERVAL '10' SECOND) as w_id; diff --git a/tests/queries/0_stateless/02127_connection_drain.reference b/tests/queries/0_stateless/02127_connection_drain.reference deleted file mode 100644 index c31f2f40f6d..00000000000 --- a/tests/queries/0_stateless/02127_connection_drain.reference +++ /dev/null @@ -1,2 +0,0 @@ -OK: sync drain -OK: async drain diff --git a/tests/queries/0_stateless/02127_connection_drain.sh b/tests/queries/0_stateless/02127_connection_drain.sh deleted file mode 100755 index 523b02d9bd5..00000000000 --- a/tests/queries/0_stateless/02127_connection_drain.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bash -# Tags: no-parallel - -CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -# shellcheck source=../shell_config.sh -. "$CUR_DIR"/../shell_config.sh - -# sync drain -for _ in {1..100}; do - prev=$(curl -d@- -sS "${CLICKHOUSE_URL}" <<<"select value from system.metrics where metric = 'SyncDrainedConnections'") - curl -d@- -sS "${CLICKHOUSE_URL}" <<<"select * from remote('127.{2,3}', view(select * from numbers(1e6))) limit 100 settings drain_timeout=-1 format Null" - now=$(curl -d@- -sS "${CLICKHOUSE_URL}" <<<"select value from system.metrics where metric = 'SyncDrainedConnections'") - if [[ "$prev" != $(( now-2 )) ]]; then - continue - fi - echo "OK: sync drain" - break -done - -# async drain -for _ in {1..100}; do - prev=$(curl -d@- -sS "${CLICKHOUSE_URL}" <<<"select value from system.metrics where metric = 'AsyncDrainedConnections'") - curl -d@- -sS "${CLICKHOUSE_URL}" <<<"select * from remote('127.{2,3}', view(select * from numbers(1e6))) limit 100 settings drain_timeout=10 format Null" - now=$(curl -d@- -sS "${CLICKHOUSE_URL}" <<<"select value from system.metrics where metric = 'AsyncDrainedConnections'") - if [[ "$prev" != $(( now-2 )) ]]; then - continue - fi - echo "OK: async drain" - break -done diff --git a/tests/queries/0_stateless/02153_native_bounds_check.reference b/tests/queries/0_stateless/02153_native_bounds_check.reference deleted file mode 100644 index d00491fd7e5..00000000000 --- a/tests/queries/0_stateless/02153_native_bounds_check.reference +++ /dev/null @@ -1 +0,0 @@ -1 diff --git a/tests/queries/0_stateless/02153_native_bounds_check.sh b/tests/queries/0_stateless/02153_native_bounds_check.sh deleted file mode 100755 index a3475ddacae..00000000000 --- a/tests/queries/0_stateless/02153_native_bounds_check.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash - -CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -# shellcheck source=../shell_config.sh -. "$CURDIR"/../shell_config.sh - -# Should correctly handle error. - -${CLICKHOUSE_LOCAL} --query "SELECT toString(number) AS a, toString(number) AS a FROM numbers(10)" --output-format Native | - ${CLICKHOUSE_LOCAL} --query "SELECT * FROM table" --input-format Native --structure 'a LowCardinality(String)' 2>&1 | - grep -c -F Exception diff --git a/tests/queries/0_stateless/02160_untuple_exponential_growth.sh b/tests/queries/0_stateless/02160_untuple_exponential_growth.sh index 9ec6594af69..2bc8f74a524 100755 --- a/tests/queries/0_stateless/02160_untuple_exponential_growth.sh +++ b/tests/queries/0_stateless/02160_untuple_exponential_growth.sh @@ -7,5 +7,5 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # Should finish in reasonable time (milliseconds). # In previous versions this query led to exponential complexity of query analysis. -${CLICKHOUSE_LOCAL} --query "SELECT untuple(tuple(untuple((1, untuple((untuple(tuple(untuple(tuple(untuple((untuple((1, 1, 1, 1)), 1, 1, 1)))))), 1, 1))))))" 2>&1 | grep -cF 'TOO_BIG_AST' -${CLICKHOUSE_LOCAL} --query "SELECT untuple(tuple(untuple(tuple(untuple(tuple(untuple(tuple(untuple(tuple(untuple(tuple(untuple(tuple(untuple(tuple(untuple(tuple(untuple(tuple(untuple(tuple(untuple(tuple(untuple((1, 1, 1, 1, 1))))))))))))))))))))))))))" 2>&1 | grep -cF 'TOO_BIG_AST' +${CLICKHOUSE_LOCAL} --query "SELECT untuple(tuple(untuple((1, untuple((untuple(tuple(untuple(tuple(untuple((untuple((1, 1, 1, 1)), 1, 1, 1)))))), 1, 1))))))" 2>&1 | grep -cF 'too big' +${CLICKHOUSE_LOCAL} --query "SELECT untuple(tuple(untuple(tuple(untuple(tuple(untuple(tuple(untuple(tuple(untuple(tuple(untuple(tuple(untuple(tuple(untuple(tuple(untuple(tuple(untuple(tuple(untuple(tuple(untuple((1, 1, 1, 1, 1))))))))))))))))))))))))))" 2>&1 | grep -cF 'too big' diff --git a/tests/queries/0_stateless/02163_shard_num.reference b/tests/queries/0_stateless/02163_shard_num.reference index f93803789a1..77eea7c95b9 100644 --- a/tests/queries/0_stateless/02163_shard_num.reference +++ b/tests/queries/0_stateless/02163_shard_num.reference @@ -15,4 +15,4 @@ SELECT _shard_num AS shard_num, sum(1) as rows FROM remote('127.{1,2}', system, SELECT a._shard_num AS shard_num, sum(1) as rows FROM remote('127.{1,2}', system, one) a GROUP BY shard_num; 2 1 1 1 -SELECT _shard_num FROM remote('127.1', system.one) AS a INNER JOIN (SELECT _shard_num FROM system.one) AS b USING (dummy); -- { serverError UNSUPPORTED_METHOD } +SELECT _shard_num FROM remote('127.1', system.one) AS a INNER JOIN (SELECT _shard_num FROM system.one) AS b USING (dummy); -- { serverError UNSUPPORTED_METHOD, UNKNOWN_IDENTIFIER } diff --git a/tests/queries/0_stateless/02163_shard_num.sql b/tests/queries/0_stateless/02163_shard_num.sql index 65a2a8ca7c2..cc87140ebaf 100644 --- a/tests/queries/0_stateless/02163_shard_num.sql +++ b/tests/queries/0_stateless/02163_shard_num.sql @@ -1,5 +1,3 @@ -SET allow_experimental_analyzer = 1; - -- { echoOn } SELECT shardNum() AS shard_num, sum(1) as rows FROM remote('127.{1,2}', system, one) GROUP BY _shard_num; @@ -7,6 +5,6 @@ SELECT shardNum() AS shard_num, sum(1) as rows FROM remote('127.{1,2}', system, SELECT _shard_num AS shard_num, sum(1) as rows FROM remote('127.{1,2}', system, one) GROUP BY _shard_num; SELECT _shard_num AS shard_num, sum(1) as rows FROM remote('127.{1,2}', system, one) GROUP BY shard_num; SELECT a._shard_num AS shard_num, sum(1) as rows FROM remote('127.{1,2}', system, one) a GROUP BY shard_num; -SELECT _shard_num FROM remote('127.1', system.one) AS a INNER JOIN (SELECT _shard_num FROM system.one) AS b USING (dummy); -- { serverError UNSUPPORTED_METHOD } +SELECT _shard_num FROM remote('127.1', system.one) AS a INNER JOIN (SELECT _shard_num FROM system.one) AS b USING (dummy); -- { serverError UNSUPPORTED_METHOD, UNKNOWN_IDENTIFIER } -- { echoOff } diff --git a/tests/queries/0_stateless/02174_cte_scalar_cache.reference b/tests/queries/0_stateless/02174_cte_scalar_cache.reference index 817116eda88..1acbef35325 100644 --- a/tests/queries/0_stateless/02174_cte_scalar_cache.reference +++ b/tests/queries/0_stateless/02174_cte_scalar_cache.reference @@ -1,3 +1,3 @@ -02177_CTE_GLOBAL_ON 5 500 11 0 5 -02177_CTE_GLOBAL_OFF 1 100 5 0 1 -02177_CTE_NEW_ANALYZER 2 200 3 0 2 +02177_CTE_GLOBAL_ON 1 100 4 0 1 +02177_CTE_GLOBAL_OFF 1 100 4 0 1 +02177_CTE_NEW_ANALYZER 1 100 4 0 1 diff --git a/tests/queries/0_stateless/02174_cte_scalar_cache.sql b/tests/queries/0_stateless/02174_cte_scalar_cache.sql index 9ed80d08cff..50a10834e64 100644 --- a/tests/queries/0_stateless/02174_cte_scalar_cache.sql +++ b/tests/queries/0_stateless/02174_cte_scalar_cache.sql @@ -1,3 +1,5 @@ +SET allow_experimental_analyzer = 1; + WITH ( SELECT sleep(0.0001) FROM system.one ) as a1, ( SELECT sleep(0.0001) FROM system.one ) as a2, diff --git a/tests/queries/0_stateless/02179_sparse_columns_detach.reference b/tests/queries/0_stateless/02179_sparse_columns_detach.reference index 2f9714f7a97..04a9b10c09f 100644 --- a/tests/queries/0_stateless/02179_sparse_columns_detach.reference +++ b/tests/queries/0_stateless/02179_sparse_columns_detach.reference @@ -1,12 +1,12 @@ -1000 +954 id Default s Sparse -1000 +954 id Default s Sparse -1000 +954 id Default s Sparse -1000 +954 id Default s Sparse diff --git a/tests/queries/0_stateless/02179_sparse_columns_detach.sql b/tests/queries/0_stateless/02179_sparse_columns_detach.sql index 4720e6720ba..2ae088fedb4 100644 --- a/tests/queries/0_stateless/02179_sparse_columns_detach.sql +++ b/tests/queries/0_stateless/02179_sparse_columns_detach.sql @@ -4,8 +4,8 @@ CREATE TABLE t_sparse_detach(id UInt64, s String) ENGINE = MergeTree ORDER BY id SETTINGS ratio_of_defaults_for_sparse_serialization = 0.9; -INSERT INTO t_sparse_detach SELECT number, number % 20 = 0 ? toString(number) : '' FROM numbers(10000); -INSERT INTO t_sparse_detach SELECT number, number % 20 = 0 ? toString(number) : '' FROM numbers(10000); +INSERT INTO t_sparse_detach SELECT number, number % 21 = 0 ? toString(number) : '' FROM numbers(10000); +INSERT INTO t_sparse_detach SELECT number, number % 21 = 0 ? toString(number) : '' FROM numbers(10000); OPTIMIZE TABLE t_sparse_detach FINAL; @@ -30,8 +30,8 @@ ALTER TABLE t_sparse_detach MODIFY SETTING vertical_merge_algorithm_min_rows_to_activate = 1, vertical_merge_algorithm_min_columns_to_activate = 1; -INSERT INTO t_sparse_detach SELECT number, number % 20 = 0 ? toString(number) : '' FROM numbers(10000); -INSERT INTO t_sparse_detach SELECT number, number % 20 = 0 ? toString(number) : '' FROM numbers(10000); +INSERT INTO t_sparse_detach SELECT number, number % 21 = 0 ? toString(number) : '' FROM numbers(10000); +INSERT INTO t_sparse_detach SELECT number, number % 21 = 0 ? toString(number) : '' FROM numbers(10000); OPTIMIZE TABLE t_sparse_detach FINAL; diff --git a/tests/queries/0_stateless/02184_default_table_engine.sql b/tests/queries/0_stateless/02184_default_table_engine.sql index 4b5ad6c008c..109875d53a5 100644 --- a/tests/queries/0_stateless/02184_default_table_engine.sql +++ b/tests/queries/0_stateless/02184_default_table_engine.sql @@ -82,7 +82,7 @@ SET default_table_engine = 'Log'; CREATE TEMPORARY TABLE tmp (n int); SHOW CREATE TEMPORARY TABLE tmp; CREATE TEMPORARY TABLE tmp1 (n int) ENGINE=Memory; -CREATE TEMPORARY TABLE tmp2 (n int) ENGINE=Log; -- {serverError 80} +CREATE TEMPORARY TABLE tmp2 (n int) ENGINE=Log; CREATE TEMPORARY TABLE tmp2 (n int) ORDER BY n; -- {serverError 80} CREATE TEMPORARY TABLE tmp2 (n int, PRIMARY KEY (n)); -- {serverError 80} diff --git a/tests/queries/0_stateless/02184_hash_functions_and_ip_types.reference b/tests/queries/0_stateless/02184_hash_functions_and_ip_types.reference index 07705827428..b305806cd08 100644 --- a/tests/queries/0_stateless/02184_hash_functions_and_ip_types.reference +++ b/tests/queries/0_stateless/02184_hash_functions_and_ip_types.reference @@ -1,54 +1,54 @@ Row 1: ────── -ipv4: 1.2.3.4 -halfMD5(toIPv4('1.2.3.4')): 14356538739656272800 -farmFingerprint64(toIPv4('1.2.3.4')): 5715546585361069049 -xxh3(toIPv4('1.2.3.4')): 14355428563589734825 -wyHash64(toIPv4('1.2.3.4')): 13096729196120951355 -xxHash32(toIPv4('1.2.3.4')): 2430391091 -gccMurmurHash(toIPv4('1.2.3.4')): 5478801830569062645 -murmurHash2_32(toIPv4('1.2.3.4')): 1658978282 -javaHashUTF16LE(toIPv4('1.2.3.4')): 24190 -intHash64(toIPv4('1.2.3.4')): 5715546585361069049 -intHash32(toIPv4('1.2.3.4')): 3152671896 -metroHash64(toIPv4('1.2.3.4')): 5715546585361069049 -hex(murmurHash3_128(toIPv4('1.2.3.4'))): 549E9EF692591F6BB55874EF9A0DE88E -jumpConsistentHash(toIPv4('1.2.3.4'), 42): 37 -sipHash64(toIPv4('1.2.3.4')): 10711397536826262068 -hex(sipHash128(toIPv4('1.2.3.4'))): DBB6A76B92B59789EFB42185DC32311D -kostikConsistentHash(toIPv4('1.2.3.4'), 42): 0 -xxHash64(toIPv4('1.2.3.4')): 14496144933713060978 -murmurHash2_64(toIPv4('1.2.3.4')): 10829690723193326442 -cityHash64(toIPv4('1.2.3.4')): 5715546585361069049 -hiveHash(toIPv4('1.2.3.4')): 122110 -murmurHash3_64(toIPv4('1.2.3.4')): 16570805747704317665 -murmurHash3_32(toIPv4('1.2.3.4')): 1165084099 -yandexConsistentHash(toIPv4('1.2.3.4'), 42): 0 +ipv4: 1.2.3.4 +halfMD5(ipv4): 14356538739656272800 +farmFingerprint64(ipv4): 5715546585361069049 +xxh3(ipv4): 14355428563589734825 +wyHash64(ipv4): 13096729196120951355 +xxHash32(ipv4): 2430391091 +gccMurmurHash(ipv4): 5478801830569062645 +murmurHash2_32(ipv4): 1658978282 +javaHashUTF16LE(ipv4): 24190 +intHash64(ipv4): 5715546585361069049 +intHash32(ipv4): 3152671896 +metroHash64(ipv4): 5715546585361069049 +hex(murmurHash3_128(ipv4)): 549E9EF692591F6BB55874EF9A0DE88E +jumpConsistentHash(ipv4, 42): 37 +sipHash64(ipv4): 10711397536826262068 +hex(sipHash128(ipv4)): DBB6A76B92B59789EFB42185DC32311D +kostikConsistentHash(ipv4, 42): 0 +xxHash64(ipv4): 14496144933713060978 +murmurHash2_64(ipv4): 10829690723193326442 +cityHash64(ipv4): 5715546585361069049 +hiveHash(ipv4): 122110 +murmurHash3_64(ipv4): 16570805747704317665 +murmurHash3_32(ipv4): 1165084099 +yandexConsistentHash(ipv4, 42): 0 Row 1: ────── -ipv6: fe80::62:5aff:fed1:daf0 -halfMD5(toIPv6('fe80::62:5aff:fed1:daf0')): 9503062220758009199 -hex(MD4(toIPv6('fe80::62:5aff:fed1:daf0'))): E35A1A4FB3A3953421AB348B2E1A4A1A -hex(MD5(toIPv6('fe80::62:5aff:fed1:daf0'))): 83E1A8BD8AB7456FC229208409F79798 -hex(SHA1(toIPv6('fe80::62:5aff:fed1:daf0'))): A6D5DCE882AC44804382DE4639E6001612E1C8B5 -hex(SHA224(toIPv6('fe80::62:5aff:fed1:daf0'))): F6995FD7BED2BCA21F68DAC6BBABE742DC1BA177BA8594CEF1715C52 -hex(SHA256(toIPv6('fe80::62:5aff:fed1:daf0'))): F75497BAD6F7747BD6B150B6F69BA2DEE354F1C2A34B7BEA6183973B78640250 -hex(SHA512(toIPv6('fe80::62:5aff:fed1:daf0'))): 0C2893CCBF44BC19CCF339AEED5B68CBFD5A2EF38263A48FE21C3379BA4438E7FF7A02F59D7542442C6E6ED538E6D13D65D3573DADB381651D3D8A5DEA232EAC -farmFingerprint64(toIPv6('fe80::62:5aff:fed1:daf0')): 6643158734288374888 -javaHash(toIPv6('fe80::62:5aff:fed1:daf0')): 684606770 -xxh3(toIPv6('fe80::62:5aff:fed1:daf0')): 4051340969481364358 -wyHash64(toIPv6('fe80::62:5aff:fed1:daf0')): 18071806066582739916 -xxHash32(toIPv6('fe80::62:5aff:fed1:daf0')): 3353862080 -gccMurmurHash(toIPv6('fe80::62:5aff:fed1:daf0')): 11049311547848936878 -murmurHash2_32(toIPv6('fe80::62:5aff:fed1:daf0')): 1039121047 -javaHashUTF16LE(toIPv6('fe80::62:5aff:fed1:daf0')): -666938696 -metroHash64(toIPv6('fe80::62:5aff:fed1:daf0')): 15333045864940909774 -hex(sipHash128(toIPv6('fe80::62:5aff:fed1:daf0'))): 31D50562F877B1F92A99B05B646568B7 -hex(murmurHash3_128(toIPv6('fe80::62:5aff:fed1:daf0'))): 6FFEF0C1DF8B5B472FE2EDF0C76C12B9 -sipHash64(toIPv6('fe80::62:5aff:fed1:daf0')): 5681592867096972315 -xxHash64(toIPv6('fe80::62:5aff:fed1:daf0')): 4533874364641685764 -murmurHash2_64(toIPv6('fe80::62:5aff:fed1:daf0')): 11839090601505681839 -cityHash64(toIPv6('fe80::62:5aff:fed1:daf0')): 1599722731594796935 -hiveHash(toIPv6('fe80::62:5aff:fed1:daf0')): 684606770 -murmurHash3_64(toIPv6('fe80::62:5aff:fed1:daf0')): 18323430650022796352 -murmurHash3_32(toIPv6('fe80::62:5aff:fed1:daf0')): 3971193740 +ipv6: fe80::62:5aff:fed1:daf0 +halfMD5(ipv6): 9503062220758009199 +hex(MD4(ipv6)): E35A1A4FB3A3953421AB348B2E1A4A1A +hex(MD5(ipv6)): 83E1A8BD8AB7456FC229208409F79798 +hex(SHA1(ipv6)): A6D5DCE882AC44804382DE4639E6001612E1C8B5 +hex(SHA224(ipv6)): F6995FD7BED2BCA21F68DAC6BBABE742DC1BA177BA8594CEF1715C52 +hex(SHA256(ipv6)): F75497BAD6F7747BD6B150B6F69BA2DEE354F1C2A34B7BEA6183973B78640250 +hex(SHA512(ipv6)): 0C2893CCBF44BC19CCF339AEED5B68CBFD5A2EF38263A48FE21C3379BA4438E7FF7A02F59D7542442C6E6ED538E6D13D65D3573DADB381651D3D8A5DEA232EAC +farmFingerprint64(ipv6): 6643158734288374888 +javaHash(ipv6): 684606770 +xxh3(ipv6): 4051340969481364358 +wyHash64(ipv6): 18071806066582739916 +xxHash32(ipv6): 3353862080 +gccMurmurHash(ipv6): 11049311547848936878 +murmurHash2_32(ipv6): 1039121047 +javaHashUTF16LE(ipv6): -666938696 +metroHash64(ipv6): 15333045864940909774 +hex(sipHash128(ipv6)): 31D50562F877B1F92A99B05B646568B7 +hex(murmurHash3_128(ipv6)): 6FFEF0C1DF8B5B472FE2EDF0C76C12B9 +sipHash64(ipv6): 5681592867096972315 +xxHash64(ipv6): 4533874364641685764 +murmurHash2_64(ipv6): 11839090601505681839 +cityHash64(ipv6): 1599722731594796935 +hiveHash(ipv6): 684606770 +murmurHash3_64(ipv6): 18323430650022796352 +murmurHash3_32(ipv6): 3971193740 diff --git a/tests/queries/0_stateless/02184_hash_functions_and_ip_types.sql b/tests/queries/0_stateless/02184_hash_functions_and_ip_types.sql index 67aae812144..d96574ef4fe 100644 --- a/tests/queries/0_stateless/02184_hash_functions_and_ip_types.sql +++ b/tests/queries/0_stateless/02184_hash_functions_and_ip_types.sql @@ -1,5 +1,7 @@ -- Tags: no-fasttest +SET allow_experimental_analyzer = 1; + SELECT toIPv4('1.2.3.4') AS ipv4, halfMD5(ipv4), diff --git a/tests/queries/0_stateless/02205_HTTP_user_agent.python b/tests/queries/0_stateless/02205_HTTP_user_agent.python index 397e06cbe82..0d3a563c094 100644 --- a/tests/queries/0_stateless/02205_HTTP_user_agent.python +++ b/tests/queries/0_stateless/02205_HTTP_user_agent.python @@ -125,7 +125,7 @@ def test_select(): def main(): # HEAD + GET - t = start_server(2) + t = start_server(3) t.start() test_select() t.join() diff --git a/tests/queries/0_stateless/02226_analyzer_or_like_combine.reference b/tests/queries/0_stateless/02226_analyzer_or_like_combine.reference index 6165079994f..d741391067c 100644 --- a/tests/queries/0_stateless/02226_analyzer_or_like_combine.reference +++ b/tests/queries/0_stateless/02226_analyzer_or_like_combine.reference @@ -78,17 +78,17 @@ SELECT materialize(\'Привет, World\') AS s1, materialize(\'Привет, World\') AS s2 WHERE (s1 LIKE \'hell%\') OR (s2 ILIKE \'%привет%\') OR (s1 ILIKE \'world%\') -SETTINGS optimize_or_like_chain = 1 +SETTINGS optimize_or_like_chain = 1, allow_hyperscan = 0 SELECT materialize(\'Привет, World\') AS s1, materialize(\'Привет, World\') AS s2 WHERE (s1 LIKE \'hell%\') OR (s2 ILIKE \'%привет%\') OR (s1 ILIKE \'world%\') -SETTINGS optimize_or_like_chain = 1 +SETTINGS optimize_or_like_chain = 1, max_hyperscan_regexp_length = 10 SELECT materialize(\'Привет, World\') AS s1, materialize(\'Привет, World\') AS s2 WHERE (s1 LIKE \'hell%\') OR (s2 ILIKE \'%привет%\') OR (s1 ILIKE \'world%\') -SETTINGS optimize_or_like_chain = 1 +SETTINGS optimize_or_like_chain = 1, max_hyperscan_regexp_total_length = 10 SELECT materialize(\'Привет, World\') AS s1, materialize(\'Привет, World\') AS s2 diff --git a/tests/queries/0_stateless/02226_analyzer_or_like_combine.sql b/tests/queries/0_stateless/02226_analyzer_or_like_combine.sql index dec73c201ef..fbebfc6d281 100644 --- a/tests/queries/0_stateless/02226_analyzer_or_like_combine.sql +++ b/tests/queries/0_stateless/02226_analyzer_or_like_combine.sql @@ -4,9 +4,9 @@ EXPLAIN SYNTAX SELECT materialize('Привет, World') AS s WHERE (s LIKE 'hel EXPLAIN QUERY TREE run_passes=1 SELECT materialize('Привет, World') AS s WHERE (s LIKE 'hell%') OR (s ILIKE '%привет%') OR (s ILIKE 'world%') SETTINGS optimize_or_like_chain = 1, allow_experimental_analyzer = 1; EXPLAIN SYNTAX SELECT materialize('Привет, World') AS s1, materialize('Привет, World') AS s2 WHERE (s1 LIKE 'hell%') OR (s2 ILIKE '%привет%') OR (s1 ILIKE 'world%') SETTINGS optimize_or_like_chain = 1; -EXPLAIN SYNTAX SELECT materialize('Привет, World') AS s1, materialize('Привет, World') AS s2 WHERE (s1 LIKE 'hell%') OR (s2 ILIKE '%привет%') OR (s1 ILIKE 'world%') SETTINGS optimize_or_like_chain = 1 SETTINGS allow_hyperscan = 0; -EXPLAIN SYNTAX SELECT materialize('Привет, World') AS s1, materialize('Привет, World') AS s2 WHERE (s1 LIKE 'hell%') OR (s2 ILIKE '%привет%') OR (s1 ILIKE 'world%') SETTINGS optimize_or_like_chain = 1 SETTINGS max_hyperscan_regexp_length = 10; -EXPLAIN SYNTAX SELECT materialize('Привет, World') AS s1, materialize('Привет, World') AS s2 WHERE (s1 LIKE 'hell%') OR (s2 ILIKE '%привет%') OR (s1 ILIKE 'world%') SETTINGS optimize_or_like_chain = 1 SETTINGS max_hyperscan_regexp_total_length = 10; +EXPLAIN SYNTAX SELECT materialize('Привет, World') AS s1, materialize('Привет, World') AS s2 WHERE (s1 LIKE 'hell%') OR (s2 ILIKE '%привет%') OR (s1 ILIKE 'world%') SETTINGS optimize_or_like_chain = 1, allow_hyperscan = 0; +EXPLAIN SYNTAX SELECT materialize('Привет, World') AS s1, materialize('Привет, World') AS s2 WHERE (s1 LIKE 'hell%') OR (s2 ILIKE '%привет%') OR (s1 ILIKE 'world%') SETTINGS optimize_or_like_chain = 1, max_hyperscan_regexp_length = 10; +EXPLAIN SYNTAX SELECT materialize('Привет, World') AS s1, materialize('Привет, World') AS s2 WHERE (s1 LIKE 'hell%') OR (s2 ILIKE '%привет%') OR (s1 ILIKE 'world%') SETTINGS optimize_or_like_chain = 1, max_hyperscan_regexp_total_length = 10; EXPLAIN SYNTAX SELECT materialize('Привет, World') AS s1, materialize('Привет, World') AS s2 WHERE (s1 LIKE 'hell%') OR (s2 ILIKE '%привет%') OR (s1 ILIKE 'world%') OR s1 == 'Привет' SETTINGS optimize_or_like_chain = 1; diff --git a/tests/queries/0_stateless/02227_union_match_by_name.reference b/tests/queries/0_stateless/02227_union_match_by_name.reference index cebcc42dcba..e51ea983f7f 100644 --- a/tests/queries/0_stateless/02227_union_match_by_name.reference +++ b/tests/queries/0_stateless/02227_union_match_by_name.reference @@ -1,40 +1,53 @@ --- { echo } +-- { echoOn } + EXPLAIN header = 1, optimize = 0 SELECT avgWeighted(x, y) FROM (SELECT NULL, 255 AS x, 1 AS y UNION ALL SELECT y, NULL AS x, 1 AS y); -Expression (Projection) +Expression (Project names) Header: avgWeighted(x, y) Nullable(Float64) - Expression (Before ORDER BY) - Header: avgWeighted(x, y) Nullable(Float64) + Expression (Projection) + Header: avgWeighted(x_0, y_1) Nullable(Float64) Aggregating - Header: avgWeighted(x, y) Nullable(Float64) + Header: avgWeighted(x_0, y_1) Nullable(Float64) Expression (Before GROUP BY) - Header: x Nullable(UInt8) - y UInt8 - Union - Header: x Nullable(UInt8) - y UInt8 - Expression (Conversion before UNION) - Header: x Nullable(UInt8) + Header: x_0 Nullable(UInt8) + y_1 UInt8 + Expression (Change column names to column identifiers) + Header: x_0 Nullable(UInt8) + y_1 UInt8 + Union + Header: NULL Nullable(UInt8) + x Nullable(UInt8) y UInt8 - Expression (Projection) - Header: x UInt8 + Expression (Conversion before UNION) + Header: NULL Nullable(UInt8) + x Nullable(UInt8) y UInt8 - Expression (Before ORDER BY) - Header: 255 UInt8 - 1 UInt8 - dummy UInt8 - ReadFromStorage (SystemOne) - Header: dummy UInt8 - Expression (Conversion before UNION) - Header: x Nullable(UInt8) - y UInt8 - Expression (Projection) - Header: x Nullable(Nothing) + Expression (Project names) + Header: NULL Nullable(Nothing) + x UInt8 + y UInt8 + Expression (Projection) + Header: NULL_Nullable(Nothing) Nullable(Nothing) + 255_UInt8 UInt8 + 1_UInt8 UInt8 + Expression (Change column names to column identifiers) + Header: system.one.dummy_0 UInt8 + ReadFromStorage (SystemOne) + Header: dummy UInt8 + Expression (Conversion before UNION) + Header: NULL Nullable(UInt8) + x Nullable(UInt8) y UInt8 - Expression (Before ORDER BY) - Header: 1 UInt8 - NULL Nullable(Nothing) - dummy UInt8 - ReadFromStorage (SystemOne) - Header: dummy UInt8 + Expression (Project names) + Header: y UInt8 + x Nullable(Nothing) + y UInt8 + Expression (Projection) + Header: 1_UInt8 UInt8 + NULL_Nullable(Nothing) Nullable(Nothing) + 1_UInt8 UInt8 + Expression (Change column names to column identifiers) + Header: system.one.dummy_0 UInt8 + ReadFromStorage (SystemOne) + Header: dummy UInt8 SELECT avgWeighted(x, y) FROM (SELECT NULL, 255 AS x, 1 AS y UNION ALL SELECT y, NULL AS x, 1 AS y); 255 diff --git a/tests/queries/0_stateless/02227_union_match_by_name.sql b/tests/queries/0_stateless/02227_union_match_by_name.sql index cc0ab8ba5aa..6a19add1d37 100644 --- a/tests/queries/0_stateless/02227_union_match_by_name.sql +++ b/tests/queries/0_stateless/02227_union_match_by_name.sql @@ -1,3 +1,8 @@ --- { echo } +SET allow_experimental_analyzer = 1; + +-- { echoOn } + EXPLAIN header = 1, optimize = 0 SELECT avgWeighted(x, y) FROM (SELECT NULL, 255 AS x, 1 AS y UNION ALL SELECT y, NULL AS x, 1 AS y); SELECT avgWeighted(x, y) FROM (SELECT NULL, 255 AS x, 1 AS y UNION ALL SELECT y, NULL AS x, 1 AS y); + +-- { echoOff } diff --git a/tests/queries/0_stateless/02229_client_stop_multiquery_in_SIGINT.sh b/tests/queries/0_stateless/02229_client_stop_multiquery_in_SIGINT.sh index 171dcc52c9c..e5d00bc1a1c 100755 --- a/tests/queries/0_stateless/02229_client_stop_multiquery_in_SIGINT.sh +++ b/tests/queries/0_stateless/02229_client_stop_multiquery_in_SIGINT.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# Tags: no-fasttest CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh diff --git a/tests/queries/0_stateless/02232_dist_insert_send_logs_level_hung.sh b/tests/queries/0_stateless/02232_dist_insert_send_logs_level_hung.sh index 322e7e73991..734cef06214 100755 --- a/tests/queries/0_stateless/02232_dist_insert_send_logs_level_hung.sh +++ b/tests/queries/0_stateless/02232_dist_insert_send_logs_level_hung.sh @@ -49,7 +49,16 @@ insert_client_opts=( timeout 250s $CLICKHOUSE_CLIENT "${client_opts[@]}" "${insert_client_opts[@]}" -q "insert into function remote('127.2', currentDatabase(), in_02232) select * from numbers(1e6)" # Kill underlying query of remote() to make KILL faster -timeout 30s $CLICKHOUSE_CLIENT "${client_opts[@]}" -q "KILL QUERY WHERE Settings['log_comment'] = '$CLICKHOUSE_LOG_COMMENT' SYNC" --format Null +# This test is reproducing very interesting bahaviour. +# The block size is 1, so the secondary query creates InterpreterSelectQuery for each row due to pushing to the MV. +# It works extremely slow, and the initial query produces new blocks and writes them to the socket much faster +# then the secondary query can read and process them. Therefore, it fills network buffers in the kernel. +# Once a buffer in the kernel is full, send(...) blocks until the secondary query will finish processing data +# that it already has in ReadBufferFromPocoSocket and call recv. +# Or until the kernel will decide to resize the buffer (seems like it has non-trivial rules for that). +# Anyway, it may look like the initial query got stuck, but actually it did not. +# Moreover, the initial query cannot be killed at that point, so KILL QUERY ... SYNC will get "stuck" as well. +timeout 30s $CLICKHOUSE_CLIENT "${client_opts[@]}" -q "KILL QUERY WHERE query like '%INSERT INTO $CLICKHOUSE_DATABASE.in_02232%' SYNC" --format Null echo $? $CLICKHOUSE_CLIENT "${client_opts[@]}" -nm -q " diff --git a/tests/queries/0_stateless/02233_interpolate_1.sql b/tests/queries/0_stateless/02233_interpolate_1.sql index 1924ae1fcad..3d416b27f45 100644 --- a/tests/queries/0_stateless/02233_interpolate_1.sql +++ b/tests/queries/0_stateless/02233_interpolate_1.sql @@ -1,5 +1,3 @@ -SET allow_experimental_analyzer = 1; - # Test WITH FILL without INTERPOLATE SELECT n, source, inter FROM ( SELECT toFloat32(number % 10) AS n, 'original' AS source, number as inter FROM numbers(10) WHERE number % 3 = 1 @@ -38,7 +36,7 @@ SELECT n, source, inter FROM ( # Test INTERPOLATE with inconsistent column - should produce error SELECT n, source, inter FROM ( SELECT toFloat32(number % 10) AS n, 'original' AS source, number as inter FROM numbers(10) WHERE number % 3 = 1 -) ORDER BY n WITH FILL FROM 0 TO 11.51 STEP 0.5 INTERPOLATE (inter AS source); -- { serverError 32 } +) ORDER BY n WITH FILL FROM 0 TO 11.51 STEP 0.5 INTERPOLATE (inter AS source); -- { serverError 6, 32 } # Test INTERPOLATE with aliased column SELECT n, source, inter + 1 AS inter_p FROM ( diff --git a/tests/queries/0_stateless/02286_drop_filesystem_cache.sh b/tests/queries/0_stateless/02286_drop_filesystem_cache.sh index b563c487646..991622446b8 100755 --- a/tests/queries/0_stateless/02286_drop_filesystem_cache.sh +++ b/tests/queries/0_stateless/02286_drop_filesystem_cache.sh @@ -16,7 +16,7 @@ for STORAGE_POLICY in 's3_cache' 'local_cache'; do ORDER BY key SETTINGS storage_policy='$STORAGE_POLICY', min_bytes_for_wide_part = 10485760" - $CLICKHOUSE_CLIENT --query "SYSTEM STOP MERGES" + $CLICKHOUSE_CLIENT --query "SYSTEM STOP MERGES test_02286" $CLICKHOUSE_CLIENT --query "SYSTEM DROP FILESYSTEM CACHE" $CLICKHOUSE_CLIENT --query "SELECT count() FROM system.filesystem_cache" diff --git a/tests/queries/0_stateless/02293_grouping_function.sql b/tests/queries/0_stateless/02293_grouping_function.sql index cf076c8e51c..c858eae269d 100644 --- a/tests/queries/0_stateless/02293_grouping_function.sql +++ b/tests/queries/0_stateless/02293_grouping_function.sql @@ -1,3 +1,5 @@ +set optimize_group_by_function_keys=0; + SELECT number, grouping(number, number % 2, number % 3) AS gr diff --git a/tests/queries/0_stateless/02293_grouping_function_group_by.sql b/tests/queries/0_stateless/02293_grouping_function_group_by.sql index d438a8a5277..da6477a1822 100644 --- a/tests/queries/0_stateless/02293_grouping_function_group_by.sql +++ b/tests/queries/0_stateless/02293_grouping_function_group_by.sql @@ -1,3 +1,5 @@ +set optimize_group_by_function_keys=0; + SELECT number, grouping(number, number % 2, number % 3) = 6 diff --git a/tests/queries/0_stateless/02303_query_kind.reference b/tests/queries/0_stateless/02303_query_kind.reference index 163f8b0ed5e..5af8c2b743f 100644 --- a/tests/queries/0_stateless/02303_query_kind.reference +++ b/tests/queries/0_stateless/02303_query_kind.reference @@ -1,36 +1,36 @@ -clickhouse-client --query_kind secondary_query -q explain plan header=1 select toString(dummy) as dummy from system.one group by dummy -Expression ((Projection + Before ORDER BY)) +clickhouse-client --allow_experimental_analyzer=1 --query_kind secondary_query -q explain plan header=1 select toString(dummy) as dummy from system.one group by dummy +Expression ((Project names + Projection)) Header: dummy String Aggregating - Header: toString(dummy) String - Expression (Before GROUP BY) - Header: toString(dummy) String + Header: toString(system.one.dummy_0) String + Expression ((Before GROUP BY + Change column names to column identifiers)) + Header: toString(system.one.dummy_0) String ReadFromStorage (SystemOne) Header: dummy UInt8 -clickhouse-local --query_kind secondary_query -q explain plan header=1 select toString(dummy) as dummy from system.one group by dummy -Expression ((Projection + Before ORDER BY)) +clickhouse-local --allow_experimental_analyzer=1 --query_kind secondary_query -q explain plan header=1 select toString(dummy) as dummy from system.one group by dummy +Expression ((Project names + Projection)) Header: dummy String Aggregating - Header: toString(dummy) String - Expression (Before GROUP BY) - Header: toString(dummy) String + Header: toString(system.one.dummy_0) String + Expression ((Before GROUP BY + Change column names to column identifiers)) + Header: toString(system.one.dummy_0) String ReadFromStorage (SystemOne) Header: dummy UInt8 -clickhouse-client --query_kind initial_query -q explain plan header=1 select toString(dummy) as dummy from system.one group by dummy -Expression ((Projection + Before ORDER BY)) +clickhouse-client --allow_experimental_analyzer=1 --query_kind initial_query -q explain plan header=1 select toString(dummy) as dummy from system.one group by dummy +Expression ((Project names + Projection)) Header: dummy String Aggregating - Header: toString(dummy) String - Expression (Before GROUP BY) - Header: toString(dummy) String + Header: toString(system.one.dummy_0) String + Expression ((Before GROUP BY + Change column names to column identifiers)) + Header: toString(system.one.dummy_0) String ReadFromStorage (SystemOne) Header: dummy UInt8 -clickhouse-local --query_kind initial_query -q explain plan header=1 select toString(dummy) as dummy from system.one group by dummy -Expression ((Projection + Before ORDER BY)) +clickhouse-local --allow_experimental_analyzer=1 --query_kind initial_query -q explain plan header=1 select toString(dummy) as dummy from system.one group by dummy +Expression ((Project names + Projection)) Header: dummy String Aggregating - Header: toString(dummy) String - Expression (Before GROUP BY) - Header: toString(dummy) String + Header: toString(system.one.dummy_0) String + Expression ((Before GROUP BY + Change column names to column identifiers)) + Header: toString(system.one.dummy_0) String ReadFromStorage (SystemOne) Header: dummy UInt8 diff --git a/tests/queries/0_stateless/02303_query_kind.sh b/tests/queries/0_stateless/02303_query_kind.sh index 5ad5f9ec6f4..1d883a2dcc7 100755 --- a/tests/queries/0_stateless/02303_query_kind.sh +++ b/tests/queries/0_stateless/02303_query_kind.sh @@ -4,6 +4,10 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh +opts=( + "--allow_experimental_analyzer=1" +) + function run_query() { echo "clickhouse-client $*" @@ -12,5 +16,5 @@ function run_query() echo "clickhouse-local $*" $CLICKHOUSE_LOCAL "$@" } -run_query --query_kind secondary_query -q "explain plan header=1 select toString(dummy) as dummy from system.one group by dummy" -run_query --query_kind initial_query -q "explain plan header=1 select toString(dummy) as dummy from system.one group by dummy" +run_query "${opts[@]}" --query_kind secondary_query -q "explain plan header=1 select toString(dummy) as dummy from system.one group by dummy" +run_query "${opts[@]}" --query_kind initial_query -q "explain plan header=1 select toString(dummy) as dummy from system.one group by dummy" diff --git a/tests/queries/0_stateless/02315_grouping_constant_folding.reference b/tests/queries/0_stateless/02315_grouping_constant_folding.reference index 6e591de2661..31816318a42 100644 --- a/tests/queries/0_stateless/02315_grouping_constant_folding.reference +++ b/tests/queries/0_stateless/02315_grouping_constant_folding.reference @@ -27,3 +27,17 @@ SELECT count() AS amount, a, b, GROUPING(a, b) FROM test02315 GROUP BY ROLLUP(a, 5 0 0 2 5 1 0 2 10 0 0 0 +SELECT count() AS amount, a, b, GROUPING(a, b) FROM test02315 GROUP BY GROUPING SETS ((a, b), (a, a), ()) ORDER BY (amount, a, b) SETTINGS force_grouping_standard_compatibility=0, allow_experimental_analyzer=1; +1 0 0 3 +1 0 2 3 +1 0 4 3 +1 0 6 3 +1 0 8 3 +1 1 1 3 +1 1 3 3 +1 1 5 3 +1 1 7 3 +1 1 9 3 +5 0 0 2 +5 1 0 2 +10 0 0 0 diff --git a/tests/queries/0_stateless/02315_grouping_constant_folding.sql b/tests/queries/0_stateless/02315_grouping_constant_folding.sql index ff259b7be79..f992aa0da32 100644 --- a/tests/queries/0_stateless/02315_grouping_constant_folding.sql +++ b/tests/queries/0_stateless/02315_grouping_constant_folding.sql @@ -9,5 +9,7 @@ SELECT count() AS amount, a, b, GROUPING(a, b) FROM test02315 GROUP BY GROUPING SELECT count() AS amount, a, b, GROUPING(a, b) FROM test02315 GROUP BY ROLLUP(a, b) ORDER BY (amount, a, b) SETTINGS force_grouping_standard_compatibility=0; +SELECT count() AS amount, a, b, GROUPING(a, b) FROM test02315 GROUP BY GROUPING SETS ((a, b), (a, a), ()) ORDER BY (amount, a, b) SETTINGS force_grouping_standard_compatibility=0, allow_experimental_analyzer=1; + -- { echoOff } DROP TABLE test02315; diff --git a/tests/queries/0_stateless/02342_window_view_different_struct.sql b/tests/queries/0_stateless/02342_window_view_different_struct.sql index c5bf8899cae..a5b2b8daa5a 100644 --- a/tests/queries/0_stateless/02342_window_view_different_struct.sql +++ b/tests/queries/0_stateless/02342_window_view_different_struct.sql @@ -1,3 +1,4 @@ +SET allow_experimental_analyzer = 0; SET allow_experimental_window_view = 1; DROP TABLE IF EXISTS data_02342; diff --git a/tests/queries/0_stateless/02343_group_by_use_nulls.sql b/tests/queries/0_stateless/02343_group_by_use_nulls.sql index a979a78be0d..e1d4021a943 100644 --- a/tests/queries/0_stateless/02343_group_by_use_nulls.sql +++ b/tests/queries/0_stateless/02343_group_by_use_nulls.sql @@ -1,3 +1,4 @@ +set optimize_group_by_function_keys=0; -- { echoOn } SELECT number, number % 2, sum(number) AS val FROM numbers(10) diff --git a/tests/queries/0_stateless/02345_implicit_transaction.reference b/tests/queries/0_stateless/02345_implicit_transaction.reference index e4dd35600f7..fb4254ec5a7 100644 --- a/tests/queries/0_stateless/02345_implicit_transaction.reference +++ b/tests/queries/0_stateless/02345_implicit_transaction.reference @@ -12,3 +12,6 @@ in_transaction 10000 out_transaction 0 {"'implicit_True'":"implicit_True","all":"2","is_empty":0} {"'implicit_False'":"implicit_False","all":"2","is_empty":1} +0 +0 +0 diff --git a/tests/queries/0_stateless/02345_implicit_transaction.sql b/tests/queries/0_stateless/02345_implicit_transaction.sql index e3f9cca37d1..b0cb4ab6305 100644 --- a/tests/queries/0_stateless/02345_implicit_transaction.sql +++ b/tests/queries/0_stateless/02345_implicit_transaction.sql @@ -1,4 +1,4 @@ --- Tags: no-ordinary-database +-- Tags: no-ordinary-database, no-fasttest CREATE TABLE landing (n Int64) engine=MergeTree order by n; CREATE TABLE target (n Int64) engine=MergeTree order by n; @@ -92,3 +92,13 @@ WHERE query LIKE '-- Verify that the transaction_id column is NOT populated without transaction%' GROUP BY transaction_id FORMAT JSONEachRow; + +SET implicit_transaction=1; +SET throw_on_unsupported_query_inside_transaction=1; +SELECT * FROM system.one; +SELECT * FROM cluster('test_cluster_interserver_secret', system, one); -- { serverError NOT_IMPLEMENTED } +SELECT * FROM cluster('test_cluster_two_shards', system, one); -- { serverError NOT_IMPLEMENTED } +SET throw_on_unsupported_query_inside_transaction=0; +-- there's not session in the interserver mode +SELECT * FROM cluster('test_cluster_interserver_secret', system, one) FORMAT Null; -- { serverError INVALID_TRANSACTION } +SELECT * FROM cluster('test_cluster_two_shards', system, one); diff --git a/tests/queries/0_stateless/02346_full_text_search.reference b/tests/queries/0_stateless/02346_full_text_search.reference index f1e21e511d0..9cd09110608 100644 --- a/tests/queries/0_stateless/02346_full_text_search.reference +++ b/tests/queries/0_stateless/02346_full_text_search.reference @@ -1,3 +1,4 @@ +Test inverted(2) af inverted 1 101 Alick a01 @@ -10,6 +11,7 @@ af inverted 113 Click b03 118 Click b08 1 +Test inverted() af inverted 101 Alick a01 106 Alick a06 @@ -21,9 +23,11 @@ af inverted 101 Alick a01 111 Alick b01 1 +Test on array columns af inverted 3 ['Click a03','Click b03'] 1 +Test on map columns af inverted 103 {'Click':'Click a03'} 108 {'Click':'Click a08'} @@ -32,20 +36,25 @@ af inverted 1 103 {'Click':'Click a03'} 1 +Test inverted(2) on a column with two parts af inverted 101 Alick a01 111 Alick b01 201 rick c01 1 +Test inverted(2) on UTF-8 data af inverted 102 clickhouse你好 1 +Test max_digestion_size_per_segment af inverted BC614E,05397FB1,6969696969898240,CF3304 1 +Test density==1 af inverted 1 1 +Test density==0.1 af inverted 1 1 diff --git a/tests/queries/0_stateless/02346_full_text_search.sql b/tests/queries/0_stateless/02346_full_text_search.sql index 2b10800e78f..ed086861f1f 100644 --- a/tests/queries/0_stateless/02346_full_text_search.sql +++ b/tests/queries/0_stateless/02346_full_text_search.sql @@ -2,7 +2,7 @@ SET allow_experimental_inverted_index = 1; SET log_queries = 1; ---------------------------------------------------- --- Test inverted(2) +SELECT 'Test inverted(2)'; DROP TABLE IF EXISTS tab; @@ -58,7 +58,7 @@ SELECT read_rows==8 from system.query_log LIMIT 1; ---------------------------------------------------- --- Test inverted() +SELECT 'Test inverted()'; DROP TABLE IF EXISTS tab_x; @@ -111,7 +111,7 @@ SELECT read_rows==4 from system.query_log LIMIT 1; ---------------------------------------------------- --- Test on array columns +SELECT 'Test on array columns'; DROP TABLE IF EXISTS tab; @@ -138,7 +138,7 @@ SELECT read_rows==2 from system.query_log LIMIT 1; ---------------------------------------------------- --- Test on map columns +SELECT 'Test on map columns'; DROP TABLE IF EXISTS tab; @@ -178,7 +178,8 @@ SELECT read_rows==8 from system.query_log LIMIT 1; ---------------------------------------------------- --- Test inverted(2) on a column with two parts +SELECT 'Test inverted(2) on a column with two parts'; + DROP TABLE IF EXISTS tab; @@ -206,7 +207,7 @@ SELECT read_rows==6 from system.query_log LIMIT 1; ---------------------------------------------------- --- Test inverted(2) on UTF-8 data +SELECT 'Test inverted(2) on UTF-8 data'; DROP TABLE IF EXISTS tab; @@ -234,7 +235,7 @@ SELECT read_rows==2 from system.query_log LIMIT 1; ---------------------------------------------------- --- Test max_digestion_size_per_segment +SELECT 'Test max_digestion_size_per_segment'; DROP TABLE IF EXISTS tab; @@ -265,7 +266,7 @@ SELECT read_rows==256 from system.query_log LIMIT 1; ---------------------------------------------------- --- Test density==1 +SELECT 'Test density==1'; DROP TABLE IF EXISTS tab; @@ -294,7 +295,7 @@ SELECT read_rows==0 from system.query_log LIMIT 1; ---------------------------------------------------- --- Test density==0.1 +SELECT 'Test density==0.1'; DROP TABLE IF EXISTS tab; diff --git a/tests/queries/0_stateless/02364_window_view_segfault.sh b/tests/queries/0_stateless/02364_window_view_segfault.sh index d03a1e5ae3e..3def22f4a9e 100755 --- a/tests/queries/0_stateless/02364_window_view_segfault.sh +++ b/tests/queries/0_stateless/02364_window_view_segfault.sh @@ -5,7 +5,11 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh -${CLICKHOUSE_CLIENT} --multiquery --multiline --query """ +opts=( + "--allow_experimental_analyzer=0" +) + +${CLICKHOUSE_CLIENT} "${opts[@]}" --multiquery --multiline --query """ DROP TABLE IF EXISTS mt ON CLUSTER test_shard_localhost; DROP TABLE IF EXISTS wv ON CLUSTER test_shard_localhost; CREATE TABLE mt ON CLUSTER test_shard_localhost (a Int32, timestamp DateTime) ENGINE=MergeTree ORDER BY tuple(); diff --git a/tests/queries/0_stateless/02368_cancel_write_into_hdfs.sh b/tests/queries/0_stateless/02368_cancel_write_into_hdfs.sh index 8262cd7eab5..65d0b3f434f 100755 --- a/tests/queries/0_stateless/02368_cancel_write_into_hdfs.sh +++ b/tests/queries/0_stateless/02368_cancel_write_into_hdfs.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash -# Tags: no-fasttest, no-stress +# Tags: no-fasttest, no-asan, no-tsan, no-msan, no-ubsan, no-debug +# FIXME https://github.com/ClickHouse/ClickHouse/issues/47207 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh diff --git a/tests/queries/0_stateless/02371_select_projection_normal_agg.sql b/tests/queries/0_stateless/02371_select_projection_normal_agg.sql index 283aec0b122..8650fb6b843 100644 --- a/tests/queries/0_stateless/02371_select_projection_normal_agg.sql +++ b/tests/queries/0_stateless/02371_select_projection_normal_agg.sql @@ -11,7 +11,8 @@ CREATE TABLE video_log ) ENGINE = MergeTree PARTITION BY toDate(datetime) -ORDER BY (user_id, device_id); +ORDER BY (user_id, device_id) +SETTINGS index_granularity = 8192, index_granularity_bytes = '10Mi'; DROP TABLE IF EXISTS rng; @@ -57,7 +58,8 @@ CREATE TABLE video_log_result ) ENGINE = MergeTree PARTITION BY toDate(hour) -ORDER BY sum_bytes; +ORDER BY sum_bytes +SETTINGS index_granularity = 8192, index_granularity_bytes = '10Mi'; INSERT INTO video_log_result SELECT toStartOfHour(datetime) AS hour, diff --git a/tests/queries/0_stateless/02381_join_dup_columns_in_plan.reference b/tests/queries/0_stateless/02381_join_dup_columns_in_plan.reference index bbf288c45d7..31a37862663 100644 --- a/tests/queries/0_stateless/02381_join_dup_columns_in_plan.reference +++ b/tests/queries/0_stateless/02381_join_dup_columns_in_plan.reference @@ -2,51 +2,51 @@ Expression Header: key String value String Join - Header: key String - value String + Header: s1.key_0 String + s2.value_1 String Expression - Header: key String + Header: s1.key_0 String ReadFromStorage Header: dummy UInt8 Union - Header: s2.key String - value String + Header: s2.key_2 String + s2.value_1 String Expression - Header: s2.key String - value String + Header: s2.key_2 String + s2.value_1 String ReadFromStorage Header: dummy UInt8 Expression - Header: s2.key String - value String + Header: s2.key_2 String + s2.value_1 String ReadFromStorage Header: dummy UInt8 Expression Header: key String value String Join - Header: key String - s2.key String - value String + Header: s1.key_0 String + s2.key_2 String + s2.value_1 String Sorting - Header: key String + Header: s1.key_0 String Expression - Header: key String + Header: s1.key_0 String ReadFromStorage Header: dummy UInt8 Sorting - Header: s2.key String - value String + Header: s2.key_2 String + s2.value_1 String Union - Header: s2.key String - value String + Header: s2.key_2 String + s2.value_1 String Expression - Header: s2.key String - value String + Header: s2.key_2 String + s2.value_1 String ReadFromStorage Header: dummy UInt8 Expression - Header: s2.key String - value String + Header: s2.key_2 String + s2.value_1 String ReadFromStorage Header: dummy UInt8 diff --git a/tests/queries/0_stateless/02381_join_dup_columns_in_plan.sql b/tests/queries/0_stateless/02381_join_dup_columns_in_plan.sql index 4ed6d965292..dfcd8c12e11 100644 --- a/tests/queries/0_stateless/02381_join_dup_columns_in_plan.sql +++ b/tests/queries/0_stateless/02381_join_dup_columns_in_plan.sql @@ -1,3 +1,4 @@ +SET allow_experimental_analyzer = 1; SET join_algorithm = 'hash'; EXPLAIN actions=0, description=0, header=1 diff --git a/tests/queries/0_stateless/02402_external_disk_mertrics.sql b/tests/queries/0_stateless/02402_external_disk_mertrics.sql index b675c05f45c..e9696eb7122 100644 --- a/tests/queries/0_stateless/02402_external_disk_mertrics.sql +++ b/tests/queries/0_stateless/02402_external_disk_mertrics.sql @@ -20,7 +20,8 @@ SET join_algorithm = 'partial_merge'; SET default_max_bytes_in_join = 0; SET max_bytes_in_join = 10000000; -SELECT number * 200000 as n, j * 2097152 FROM numbers(5) nums +SELECT n, j * 2097152 FROM +(SELECT number * 200000 as n FROM numbers(5)) nums ANY LEFT JOIN ( SELECT number * 2 AS n, number AS j FROM numbers(1000000) ) js2 USING n ORDER BY n diff --git a/tests/queries/0_stateless/02402_merge_engine_with_view.sql b/tests/queries/0_stateless/02402_merge_engine_with_view.sql index 64822784845..ae9de1426e7 100644 --- a/tests/queries/0_stateless/02402_merge_engine_with_view.sql +++ b/tests/queries/0_stateless/02402_merge_engine_with_view.sql @@ -1,7 +1,7 @@ -- #40014 -CREATE TABLE m0 (id UInt64) ENGINE=MergeTree ORDER BY id SETTINGS index_granularity = 1; +CREATE TABLE m0 (id UInt64) ENGINE=MergeTree ORDER BY id SETTINGS index_granularity = 1, ratio_of_defaults_for_sparse_serialization = 1.0; INSERT INTO m0 SELECT number FROM numbers(10); -CREATE TABLE m1 (id UInt64, s String) ENGINE=MergeTree ORDER BY id SETTINGS index_granularity = 1; +CREATE TABLE m1 (id UInt64, s String) ENGINE=MergeTree ORDER BY id SETTINGS index_granularity = 1, ratio_of_defaults_for_sparse_serialization = 1.0; INSERT INTO m1 SELECT number, 'boo' FROM numbers(10); CREATE VIEW m1v AS SELECT id FROM m1; diff --git a/tests/queries/0_stateless/02417_opentelemetry_insert_on_distributed_table.reference b/tests/queries/0_stateless/02417_opentelemetry_insert_on_distributed_table.reference index dde07d4540d..312904a3cf9 100644 --- a/tests/queries/0_stateless/02417_opentelemetry_insert_on_distributed_table.reference +++ b/tests/queries/0_stateless/02417_opentelemetry_insert_on_distributed_table.reference @@ -1,8 +1,18 @@ +===1=== {"operation_name":"void DB::DistributedSink::writeToLocal(const Cluster::ShardInfo &, const DB::Block &, size_t)","cluster":"test_cluster_two_shards_localhost","shard":"1","rows":"1","bytes":"8"} {"operation_name":"void DB::DistributedSink::writeToLocal(const Cluster::ShardInfo &, const DB::Block &, size_t)","cluster":"test_cluster_two_shards_localhost","shard":"2","rows":"1","bytes":"8"} -{"operation_name":"void DB::StorageDistributedDirectoryMonitor::processFile(const std::string &)","cluster":"test_cluster_two_shards_localhost","shard":"1","rows":"1","bytes":"8"} -{"operation_name":"void DB::StorageDistributedDirectoryMonitor::processFile(const std::string &)","cluster":"test_cluster_two_shards_localhost","shard":"2","rows":"1","bytes":"8"} +1 +===2=== +{"operation_name":"void DB::DistributedAsyncInsertDirectoryQueue::processFile(const std::string &)","cluster":"test_cluster_two_shards_localhost","shard":"1","rows":"1","bytes":"8"} +{"operation_name":"void DB::DistributedAsyncInsertDirectoryQueue::processFile(const std::string &)","cluster":"test_cluster_two_shards_localhost","shard":"2","rows":"1","bytes":"8"} +3 +2 +===3=== {"operation_name":"auto DB::DistributedSink::runWritingJob(DB::DistributedSink::JobReplica &, const DB::Block &, size_t)::(anonymous class)::operator()() const","cluster":"test_cluster_two_shards_localhost","shard":"1","rows":"1","bytes":"8"} {"operation_name":"auto DB::DistributedSink::runWritingJob(DB::DistributedSink::JobReplica &, const DB::Block &, size_t)::(anonymous class)::operator()() const","cluster":"test_cluster_two_shards_localhost","shard":"2","rows":"1","bytes":"8"} +1 +===4=== {"operation_name":"auto DB::DistributedSink::runWritingJob(DB::DistributedSink::JobReplica &, const DB::Block &, size_t)::(anonymous class)::operator()() const","cluster":"test_cluster_two_shards_localhost","shard":"1","rows":"1","bytes":"8"} {"operation_name":"auto DB::DistributedSink::runWritingJob(DB::DistributedSink::JobReplica &, const DB::Block &, size_t)::(anonymous class)::operator()() const","cluster":"test_cluster_two_shards_localhost","shard":"2","rows":"1","bytes":"8"} +3 +2 diff --git a/tests/queries/0_stateless/02417_opentelemetry_insert_on_distributed_table.sh b/tests/queries/0_stateless/02417_opentelemetry_insert_on_distributed_table.sh index 9ac5f061d4a..81f3e3f4ea6 100755 --- a/tests/queries/0_stateless/02417_opentelemetry_insert_on_distributed_table.sh +++ b/tests/queries/0_stateless/02417_opentelemetry_insert_on_distributed_table.sh @@ -42,6 +42,22 @@ ${CLICKHOUSE_CLIENT} -nq " ;" } +# +# $1 - OpenTelemetry Trace Id +# $2 - value of insert_distributed_sync +function check_span_kind() +{ +${CLICKHOUSE_CLIENT} -nq " + SYSTEM FLUSH LOGS; + + SELECT count() + FROM system.opentelemetry_span_log + WHERE finish_date >= yesterday() + AND lower(hex(trace_id)) = '${1}' + AND kind = '${2}' + ;" +} + # # Prepare tables for tests @@ -57,30 +73,46 @@ CREATE TABLE ${CLICKHOUSE_DATABASE}.local_opentelemetry (key UInt64) Engine=Merg # # test1 # +echo "===1===" trace_id=$(${CLICKHOUSE_CLIENT} -q "select lower(hex(generateUUIDv4()))"); insert $trace_id 0 1 "async-insert-writeToLocal" check_span $trace_id +# 1 HTTP SERVER spans +check_span_kind $trace_id 'SERVER' # # test2 # +echo "===2===" trace_id=$(${CLICKHOUSE_CLIENT} -q "select lower(hex(generateUUIDv4()))"); insert $trace_id 0 0 "async-insert-writeToRemote" check_span $trace_id +# 3 SERVER spans, 1 for HTTP, 2 for TCP +check_span_kind $trace_id 'SERVER' +# 2 CLIENT spans +check_span_kind $trace_id 'CLIENT' # # test3 # trace_id=$(${CLICKHOUSE_CLIENT} -q "select lower(hex(generateUUIDv4()))"); insert $trace_id 1 1 "sync-insert-writeToLocal" +echo "===3===" check_span $trace_id +# 1 HTTP SERVER spans +check_span_kind $trace_id 'SERVER' # # test4 # +echo "===4===" trace_id=$(${CLICKHOUSE_CLIENT} -q "select lower(hex(generateUUIDv4()))"); insert $trace_id 1 0 "sync-insert-writeToRemote" check_span $trace_id +# 3 SERVER spans, 1 for HTTP, 2 for TCP +check_span_kind $trace_id 'SERVER' +# 2 CLIENT spans +check_span_kind $trace_id 'CLIENT' # # Cleanup diff --git a/tests/queries/0_stateless/02420_final_setting_analyzer.reference b/tests/queries/0_stateless/02420_final_setting_analyzer.reference index ee7c2541bcf..9a03c484765 100644 --- a/tests/queries/0_stateless/02420_final_setting_analyzer.reference +++ b/tests/queries/0_stateless/02420_final_setting_analyzer.reference @@ -108,9 +108,6 @@ select left_table.id,val_left, val_middle, val_right from left_table ORDER BY left_table.id, val_left, val_middle, val_right; 1 c a c 1 c b c --- no distributed tests because it is not currently supported: --- JOIN with remote storages is unsupported. - -- Quite exotic with Merge engine DROP TABLE IF EXISTS table_to_merge_a; DROP TABLE IF EXISTS table_to_merge_b; diff --git a/tests/queries/0_stateless/02420_final_setting_analyzer.sql b/tests/queries/0_stateless/02420_final_setting_analyzer.sql index 5937e536239..14c832cfaf5 100644 --- a/tests/queries/0_stateless/02420_final_setting_analyzer.sql +++ b/tests/queries/0_stateless/02420_final_setting_analyzer.sql @@ -79,9 +79,6 @@ select left_table.id,val_left, val_middle, val_right from left_table inner join (SELECT * FROM right_table WHERE id = 1) r on middle_table.id = r.id ORDER BY left_table.id, val_left, val_middle, val_right; --- no distributed tests because it is not currently supported: --- JOIN with remote storages is unsupported. - -- Quite exotic with Merge engine DROP TABLE IF EXISTS table_to_merge_a; DROP TABLE IF EXISTS table_to_merge_b; diff --git a/tests/queries/0_stateless/02423_ddl_for_opentelemetry.sh b/tests/queries/0_stateless/02423_ddl_for_opentelemetry.sh index 6164ff97d9f..b2a5ebdd4ad 100755 --- a/tests/queries/0_stateless/02423_ddl_for_opentelemetry.sh +++ b/tests/queries/0_stateless/02423_ddl_for_opentelemetry.sh @@ -144,8 +144,8 @@ else # For Replicated database it will fail on initiator before enqueueing distributed DDL expected=0 fi -check_span $expected $trace_id "%executeDDLQueryOnCluster%" "attribute['clickhouse.cluster']='${cluster_name}'" -check_span $expected $trace_id "%DDLWorker::processTask%" +check_span $expected $trace_id "%executeDDLQueryOnCluster%" "attribute['clickhouse.cluster']='${cluster_name}' AND kind = 'PRODUCER'" +check_span $expected $trace_id "%DDLWorker::processTask%" "kind = 'CONSUMER'" if [ $cluster_name = "test_shard_localhost" ]; then # There should be two 'query' spans, one is for the HTTPHandler, the other is for the DDL executing in DDLWorker. diff --git a/tests/queries/0_stateless/02426_orc_bug.reference b/tests/queries/0_stateless/02426_orc_bug.reference index e5ad2b49289..baa88da2158 100644 Binary files a/tests/queries/0_stateless/02426_orc_bug.reference and b/tests/queries/0_stateless/02426_orc_bug.reference differ diff --git a/tests/queries/0_stateless/02426_orc_bug.sh b/tests/queries/0_stateless/02426_orc_bug.sh new file mode 100755 index 00000000000..7a7ad9f1783 --- /dev/null +++ b/tests/queries/0_stateless/02426_orc_bug.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +$CLICKHOUSE_CLIENT --query="SELECT arrayJoin([[], [1]]) FORMAT ORC SETTINGS output_format_orc_compression_method='none'" | md5sum; + diff --git a/tests/queries/0_stateless/02426_orc_bug.sql b/tests/queries/0_stateless/02426_orc_bug.sql deleted file mode 100644 index 7016f1ceb70..00000000000 --- a/tests/queries/0_stateless/02426_orc_bug.sql +++ /dev/null @@ -1,3 +0,0 @@ --- Tags: no-fasttest - -SELECT arrayJoin([[], [1]]) FORMAT ORC; diff --git a/tests/queries/0_stateless/02428_parameterized_view.reference b/tests/queries/0_stateless/02428_parameterized_view.reference index 52a31f53cc1..27d7c6d1956 100644 --- a/tests/queries/0_stateless/02428_parameterized_view.reference +++ b/tests/queries/0_stateless/02428_parameterized_view.reference @@ -35,3 +35,4 @@ ERROR 10 20 10 +10 diff --git a/tests/queries/0_stateless/02428_parameterized_view.sh b/tests/queries/0_stateless/02428_parameterized_view.sh index 6118013b665..aad3aad3b22 100755 --- a/tests/queries/0_stateless/02428_parameterized_view.sh +++ b/tests/queries/0_stateless/02428_parameterized_view.sh @@ -15,6 +15,7 @@ $CLICKHOUSE_CLIENT -q "DROP VIEW IF EXISTS test_02428_pv6" $CLICKHOUSE_CLIENT -q "DROP VIEW IF EXISTS test_02428_pv7" $CLICKHOUSE_CLIENT -q "DROP VIEW IF EXISTS test_02428_pv8" $CLICKHOUSE_CLIENT -q "DROP VIEW IF EXISTS test_02428_pv9" +$CLICKHOUSE_CLIENT -q "DROP VIEW IF EXISTS test_02428_pv10" $CLICKHOUSE_CLIENT -q "DROP VIEW IF EXISTS test_02428_v1" $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS test_02428_Catalog" $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS ${CLICKHOUSE_TEST_UNIQUE_NAME}.pv1" @@ -83,6 +84,9 @@ $CLICKHOUSE_CLIENT -q "SELECT * FROM test_02428_pv8(prices=[10,20])" $CLICKHOUSE_CLIENT -q "CREATE VIEW test_02428_pv9 AS SELECT Price FROM test_02428_Catalog WHERE Price IN (10,20) AND Quantity={quantity:UInt64} ORDER BY Price" $CLICKHOUSE_CLIENT -q "SELECT * FROM test_02428_pv9(quantity=3)" +$CLICKHOUSE_CLIENT -q "CREATE VIEW test_02428_pv10 AS SELECT Price FROM test_02428_Catalog WHERE Price={Pri:UInt64} ORDER BY Price" +$CLICKHOUSE_CLIENT -q "SELECT * FROM test_02428_pv10(Pri=10)" + $CLICKHOUSE_CLIENT -q "DROP VIEW test_02428_pv1" $CLICKHOUSE_CLIENT -q "DROP VIEW test_02428_pv2" $CLICKHOUSE_CLIENT -q "DROP VIEW test_02428_pv3" @@ -91,6 +95,7 @@ $CLICKHOUSE_CLIENT -q "DROP VIEW test_02428_pv6" $CLICKHOUSE_CLIENT -q "DROP VIEW test_02428_pv7" $CLICKHOUSE_CLIENT -q "DROP VIEW test_02428_pv8" $CLICKHOUSE_CLIENT -q "DROP VIEW test_02428_pv9" +$CLICKHOUSE_CLIENT -q "DROP VIEW test_02428_pv10" $CLICKHOUSE_CLIENT -q "DROP VIEW test_02428_v1" $CLICKHOUSE_CLIENT -q "DROP TABLE test_02428_Catalog" $CLICKHOUSE_CLIENT -q "DROP TABLE ${CLICKHOUSE_TEST_UNIQUE_NAME}.pv1" diff --git a/tests/queries/0_stateless/02432_s3_parallel_parts_cleanup.sql b/tests/queries/0_stateless/02432_s3_parallel_parts_cleanup.sql index 235a3335d9d..3688a649d5e 100644 --- a/tests/queries/0_stateless/02432_s3_parallel_parts_cleanup.sql +++ b/tests/queries/0_stateless/02432_s3_parallel_parts_cleanup.sql @@ -7,7 +7,7 @@ drop table if exists rmt2; create table rmt (n int, m int, k int) engine=ReplicatedMergeTree('/test/02432/{database}', '1') order by tuple() settings storage_policy = 's3_cache', allow_remote_fs_zero_copy_replication=1, max_part_removal_threads=10, concurrent_part_removal_threshold=1, cleanup_delay_period=1, cleanup_delay_period_random_add=1, - max_replicated_merges_in_queue=0, max_replicated_mutations_in_queue=0, min_bytes_for_compact_part=0, min_rows_for_compact_part=0; + max_replicated_merges_in_queue=0, max_replicated_mutations_in_queue=0, min_bytes_for_wide_part=0, min_rows_for_wide_part=0; insert into rmt(n, m) values (1, 42); insert into rmt(n, m) values (2, 42); @@ -37,7 +37,7 @@ select count(), sum(n), sum(m) from rmt; create table rmt2 (n int, m int, k String) engine=ReplicatedMergeTree('/test/02432/{database}', '2') order by tuple() settings storage_policy = 's3_cache', allow_remote_fs_zero_copy_replication=1, max_part_removal_threads=10, concurrent_part_removal_threshold=1, cleanup_delay_period=1, cleanup_delay_period_random_add=1, - min_bytes_for_compact_part=0, min_rows_for_compact_part=0, max_replicated_merges_in_queue=1, + min_bytes_for_wide_part=0, min_rows_for_wide_part=0, max_replicated_merges_in_queue=1, old_parts_lifetime=0; alter table rmt2 modify column k Nullable(String); diff --git a/tests/queries/0_stateless/02434_cancel_insert_when_client_dies.reference b/tests/queries/0_stateless/02434_cancel_insert_when_client_dies.reference new file mode 100644 index 00000000000..d2475419998 --- /dev/null +++ b/tests/queries/0_stateless/02434_cancel_insert_when_client_dies.reference @@ -0,0 +1,3 @@ +5000000 +5000000 +1 diff --git a/tests/queries/0_stateless/02434_cancel_insert_when_client_dies.sh b/tests/queries/0_stateless/02434_cancel_insert_when_client_dies.sh new file mode 100755 index 00000000000..85aa992d98c --- /dev/null +++ b/tests/queries/0_stateless/02434_cancel_insert_when_client_dies.sh @@ -0,0 +1,98 @@ +#!/usr/bin/env bash +# Tags: no-random-settings +# shellcheck disable=SC2009 + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +export DATA_FILE="$CLICKHOUSE_TMP/deduptest.tsv" +export TEST_MARK="02434_insert_${CLICKHOUSE_DATABASE}_" + +$CLICKHOUSE_CLIENT -q 'select * from numbers(5000000) format TSV' > $DATA_FILE +$CLICKHOUSE_CLIENT -q 'create table dedup_test(A Int64) Engine = MergeTree order by A settings non_replicated_deduplication_window=1000;' +$CLICKHOUSE_CLIENT -q "create table dedup_dist(A Int64) Engine = Distributed('test_cluster_one_shard_two_replicas', currentDatabase(), dedup_test)" + +function insert_data +{ + SETTINGS="query_id=$ID&max_insert_block_size=110000&min_insert_block_size_rows=110000" + # max_block_size=10000, so external table will contain smaller blocks that will be squashed on insert-select (more chances to catch a bug on query cancellation) + TRASH_SETTINGS="query_id=$ID&input_format_parallel_parsing=0&max_threads=1&max_insert_threads=1&max_insert_block_size=110000&max_block_size=10000&min_insert_block_size_bytes=0&min_insert_block_size_rows=110000&max_insert_block_size=110000" + TYPE=$(( RANDOM % 5 )) + + if [[ "$TYPE" -eq 0 ]]; then + # client will send 10000-rows blocks, server will squash them into 110000-rows blocks (more chances to catch a bug on query cancellation) + $CLICKHOUSE_CLIENT --max_block_size=10000 --max_insert_block_size=10000 --query_id="$ID" \ + -q 'insert into dedup_test settings max_insert_block_size=110000, min_insert_block_size_rows=110000 format TSV' < $DATA_FILE + elif [[ "$TYPE" -eq 1 ]]; then + $CLICKHOUSE_CLIENT --max_block_size=10000 --max_insert_block_size=10000 --query_id="$ID" --prefer_localhost_replica="$(( RANDOM % 2))" \ + -q 'insert into dedup_dist settings max_insert_block_size=110000, min_insert_block_size_rows=110000 format TSV' < $DATA_FILE + elif [[ "$TYPE" -eq 2 ]]; then + $CLICKHOUSE_CURL -sS -X POST --data-binary @- "$CLICKHOUSE_URL&$SETTINGS&query=insert+into+dedup_test+format+TSV" < $DATA_FILE + elif [[ "$TYPE" -eq 3 ]]; then + $CLICKHOUSE_CURL -sS -X POST -H "Transfer-Encoding: chunked" --data-binary @- "$CLICKHOUSE_URL&$SETTINGS&query=insert+into+dedup_test+format+TSV" < $DATA_FILE + else + $CLICKHOUSE_CURL -sS -F 'file=@-' "$CLICKHOUSE_URL&$TRASH_SETTINGS&file_format=TSV&file_types=UInt64" -X POST --form-string 'query=insert into dedup_test select * from file' < $DATA_FILE + fi +} + +export -f insert_data + +ID="02434_insert_init_${CLICKHOUSE_DATABASE}_$RANDOM" +insert_data +$CLICKHOUSE_CLIENT -q "system flush distributed dedup_dist" +$CLICKHOUSE_CLIENT -q 'select count() from dedup_test' + +function thread_insert +{ + # supress "Killed" messages from bash + while true; do + export ID="$TEST_MARK$RANDOM" + bash -c insert_data 2>&1| grep -Fav "Killed" + done +} + +function thread_select +{ + while true; do + $CLICKHOUSE_CLIENT -q "with (select count() from dedup_test) as c select throwIf(c != 5000000, 'Expected 5000000 rows, got ' || toString(c)) format Null" + sleep 0.$RANDOM; + done +} + +function thread_cancel +{ + while true; do + SIGNAL="INT" + if (( RANDOM % 2 )); then + SIGNAL="KILL" + fi + PID=$(grep -Fa "$TEST_MARK" /proc/*/cmdline | grep -Fav grep | grep -Eoa "/proc/[0-9]*/cmdline:" | grep -Eo "[0-9]*" | head -1) + if [ ! -z "$PID" ]; then kill -s "$SIGNAL" "$PID"; fi + sleep 0.$RANDOM; + sleep 0.$RANDOM; + sleep 0.$RANDOM; + done +} + +export -f thread_insert; +export -f thread_select; +export -f thread_cancel; + +TIMEOUT=40 + +timeout $TIMEOUT bash -c thread_insert & +timeout $TIMEOUT bash -c thread_select & +timeout $TIMEOUT bash -c thread_cancel 2> /dev/null & + +wait + +$CLICKHOUSE_CLIENT -q 'select count() from dedup_test' + +$CLICKHOUSE_CLIENT -q 'system flush logs' + +# Ensure that thread_cancel actually did something +$CLICKHOUSE_CLIENT -q "select count() > 0 from system.text_log where event_date >= yesterday() and query_id like '$TEST_MARK%' and ( + message_format_string in ('Unexpected end of file while reading chunk header of HTTP chunked data', 'Unexpected EOF, got {} of {} bytes', + 'Query was cancelled or a client has unexpectedly dropped the connection') or + message like '%Connection reset by peer%' or message like '%Broken pipe, while writing to socket%')" diff --git a/tests/queries/0_stateless/02435_rollback_cancelled_queries.reference b/tests/queries/0_stateless/02435_rollback_cancelled_queries.reference new file mode 100644 index 00000000000..2d32c17ec7c --- /dev/null +++ b/tests/queries/0_stateless/02435_rollback_cancelled_queries.reference @@ -0,0 +1,3 @@ +1000000 +0 +1 diff --git a/tests/queries/0_stateless/02435_rollback_cancelled_queries.sh b/tests/queries/0_stateless/02435_rollback_cancelled_queries.sh new file mode 100755 index 00000000000..e4aec6503a4 --- /dev/null +++ b/tests/queries/0_stateless/02435_rollback_cancelled_queries.sh @@ -0,0 +1,117 @@ +#!/usr/bin/env bash +# Tags: no-random-settings, no-ordinary-database +# shellcheck disable=SC2009 + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +export DATA_FILE="$CLICKHOUSE_TMP/deduptest.tsv" +export TEST_MARK="02435_insert_${CLICKHOUSE_DATABASE}_" +export SESSION="02435_session_${CLICKHOUSE_DATABASE}" + +$CLICKHOUSE_CLIENT -q 'select * from numbers(1000000) format TSV' > $DATA_FILE +$CLICKHOUSE_CLIENT -q 'create table dedup_test(A Int64) Engine = MergeTree order by sin(A) partition by intDiv(A, 100000)' + +function insert_data +{ + IMPLICIT=$(( RANDOM % 2 )) + SESSION_ID="${SESSION}_$RANDOM.$RANDOM.$RANDOM" + TXN_SETTINGS="session_id=$SESSION_ID&throw_on_unsupported_query_inside_transaction=0&implicit_transaction=$IMPLICIT" + BEGIN="" + COMMIT="" + SETTINGS="query_id=$ID&$TXN_SETTINGS&max_insert_block_size=110000&min_insert_block_size_rows=110000" + if [[ "$IMPLICIT" -eq 0 ]]; then + $CLICKHOUSE_CURL -sS -d 'begin transaction' "$CLICKHOUSE_URL&$TXN_SETTINGS" + SETTINGS="$SETTINGS&session_check=1" + BEGIN="begin transaction;" + COMMIT=$(echo -ne "\n\ncommit") + fi + + # max_block_size=10000, so external table will contain smaller blocks that will be squashed on insert-select (more chances to catch a bug on query cancellation) + TRASH_SETTINGS="$SETTINGS&input_format_parallel_parsing=0&max_threads=1&max_insert_threads=1&max_block_size=10000&min_insert_block_size_bytes=0" + TYPE=$(( RANDOM % 6 )) + + if [[ "$TYPE" -eq 0 ]]; then + $CLICKHOUSE_CURL -sS -X POST --data-binary @- "$CLICKHOUSE_URL&$SETTINGS&query=insert+into+dedup_test+format+TSV" < $DATA_FILE + elif [[ "$TYPE" -eq 1 ]]; then + $CLICKHOUSE_CURL -sS -X POST -H "Transfer-Encoding: chunked" --data-binary @- "$CLICKHOUSE_URL&$SETTINGS&query=insert+into+dedup_test+format+TSV" < $DATA_FILE + elif [[ "$TYPE" -eq 2 ]]; then + $CLICKHOUSE_CURL -sS -F 'file=@-' "$CLICKHOUSE_URL&$TRASH_SETTINGS&file_format=TSV&file_types=UInt64" -X POST --form-string 'query=insert into dedup_test select * from file' < $DATA_FILE + else + # client will send 1000-rows blocks, server will squash them into 110000-rows blocks (more chances to catch a bug on query cancellation) + $CLICKHOUSE_CLIENT --stacktrace --query_id="$ID" --throw_on_unsupported_query_inside_transaction=0 --implicit_transaction="$IMPLICIT" \ + --max_block_size=1000 --max_insert_block_size=1000 --multiquery -q \ + "${BEGIN}insert into dedup_test settings max_insert_block_size=110000, min_insert_block_size_rows=110000 format TSV$COMMIT" < $DATA_FILE \ + | grep -Fv "Transaction is not in RUNNING state" + fi + + if [[ "$IMPLICIT" -eq 0 ]]; then + $CLICKHOUSE_CURL -sS -d 'commit' "$CLICKHOUSE_URL&$TXN_SETTINGS&close_session=1" + fi +} + +export -f insert_data + +ID="02435_insert_init_${CLICKHOUSE_DATABASE}_$RANDOM" +insert_data +$CLICKHOUSE_CLIENT -q 'select count() from dedup_test' + +function thread_insert +{ + # supress "Killed" messages from bash + while true; do + export ID="$TEST_MARK$RANDOM" + bash -c insert_data 2>&1| grep -Fav "Killed" | grep -Fav "SESSION_IS_LOCKED" | grep -Fav "SESSION_NOT_FOUND" + done +} + +function thread_select +{ + while true; do + $CLICKHOUSE_CLIENT --implicit_transaction=1 -q "with (select count() from dedup_test) as c select throwIf(c % 1000000 != 0, 'Expected 1000000 * N rows, got ' || toString(c)) format Null" + sleep 0.$RANDOM; + done +} + +function thread_cancel +{ + while true; do + SIGNAL="INT" + if (( RANDOM % 2 )); then + SIGNAL="KILL" + fi + PID=$(grep -Fa "$TEST_MARK" /proc/*/cmdline | grep -Fav grep | grep -Eoa "/proc/[0-9]*/cmdline:" | grep -Eo "[0-9]*" | head -1) + if [ ! -z "$PID" ]; then kill -s "$SIGNAL" "$PID"; fi + sleep 0.$RANDOM; + done +} + +export -f thread_insert; +export -f thread_select; +export -f thread_cancel; + +TIMEOUT=20 + +timeout $TIMEOUT bash -c thread_insert & +timeout $TIMEOUT bash -c thread_select & +timeout $TIMEOUT bash -c thread_cancel 2> /dev/null & + +wait + +$CLICKHOUSE_CLIENT -q 'system flush logs' + +ID="02435_insert_last_${CLICKHOUSE_DATABASE}_$RANDOM" +insert_data + +$CLICKHOUSE_CLIENT --implicit_transaction=1 -q 'select throwIf(count() % 1000000 != 0 or count() = 0) from dedup_test' \ + || $CLICKHOUSE_CLIENT -q "select name, rows, active, visible, creation_tid, creation_csn from system.parts where database=currentDatabase();" + +# Ensure that thread_cancel actually did something +$CLICKHOUSE_CLIENT -q "select count() > 0 from system.text_log where event_date >= yesterday() and query_id like '$TEST_MARK%' and ( + message_format_string in ('Unexpected end of file while reading chunk header of HTTP chunked data', 'Unexpected EOF, got {} of {} bytes', + 'Query was cancelled or a client has unexpectedly dropped the connection') or + message like '%Connection reset by peer%' or message like '%Broken pipe, while writing to socket%')" + +wait_for_queries_to_finish 30 +$CLICKHOUSE_CLIENT --database_atomic_wait_for_drop_and_detach_synchronously=0 -q "drop table dedup_test" diff --git a/tests/queries/0_stateless/02451_order_by_monotonic.reference b/tests/queries/0_stateless/02451_order_by_monotonic.reference index d3de324a7e1..f9f0ef38be1 100644 --- a/tests/queries/0_stateless/02451_order_by_monotonic.reference +++ b/tests/queries/0_stateless/02451_order_by_monotonic.reference @@ -4,19 +4,19 @@ 2022-09-09 12:00:00 0x 2022-09-09 12:00:00 1 2022-09-09 12:00:00 1x - Prefix sort description: toStartOfMinute(t) ASC - Result sort description: toStartOfMinute(t) ASC, c1 ASC - Prefix sort description: toStartOfMinute(t) ASC - Result sort description: toStartOfMinute(t) ASC - Prefix sort description: negate(a) ASC - Result sort description: negate(a) ASC - Prefix sort description: negate(a) ASC, negate(b) ASC - Result sort description: negate(a) ASC, negate(b) ASC - Prefix sort description: a DESC, negate(b) ASC - Result sort description: a DESC, negate(b) ASC - Prefix sort description: negate(a) ASC, b DESC - Result sort description: negate(a) ASC, b DESC - Prefix sort description: negate(a) ASC - Result sort description: negate(a) ASC, b ASC - Prefix sort description: a ASC - Result sort description: a ASC, negate(b) ASC + Prefix sort description: toStartOfMinute(test.t_0) ASC + Result sort description: toStartOfMinute(test.t_0) ASC, test.c1_1 ASC + Prefix sort description: toStartOfMinute(test.t_0) ASC + Result sort description: toStartOfMinute(test.t_0) ASC + Prefix sort description: negate(test.a_0) ASC + Result sort description: negate(test.a_0) ASC + Prefix sort description: negate(test.a_0) ASC, negate(test.b_1) ASC + Result sort description: negate(test.a_0) ASC, negate(test.b_1) ASC + Prefix sort description: test.a_0 DESC, negate(test.b_1) ASC + Result sort description: test.a_0 DESC, negate(test.b_1) ASC + Prefix sort description: negate(test.a_0) ASC, test.b_1 DESC + Result sort description: negate(test.a_0) ASC, test.b_1 DESC + Prefix sort description: negate(test.a_0) ASC + Result sort description: negate(test.a_0) ASC, test.b_1 ASC + Prefix sort description: test.a_0 ASC + Result sort description: test.a_0 ASC, negate(test.b_1) ASC diff --git a/tests/queries/0_stateless/02451_order_by_monotonic.sh b/tests/queries/0_stateless/02451_order_by_monotonic.sh index cc26ba91e1c..7d1356b4445 100755 --- a/tests/queries/0_stateless/02451_order_by_monotonic.sh +++ b/tests/queries/0_stateless/02451_order_by_monotonic.sh @@ -4,37 +4,41 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh +opts=( + "--allow_experimental_analyzer=1" +) + function explain_sort_description() { - out=$($CLICKHOUSE_CLIENT --optimize_read_in_order=1 -q "EXPLAIN PLAN actions = 1 $1") + out=$($CLICKHOUSE_CLIENT "${opts[@]}" --optimize_read_in_order=1 -q "EXPLAIN PLAN actions = 1 $1") echo "$out" | grep "Prefix sort description:" echo "$out" | grep "Result sort description:" } -$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS t_order_by_monotonic" -$CLICKHOUSE_CLIENT -q "CREATE TABLE t_order_by_monotonic (t DateTime, c1 String) ENGINE = MergeTree ORDER BY (t, c1) +$CLICKHOUSE_CLIENT "${opts[@]}" -q "DROP TABLE IF EXISTS t_order_by_monotonic" +$CLICKHOUSE_CLIENT "${opts[@]}" -q "CREATE TABLE t_order_by_monotonic (t DateTime, c1 String) ENGINE = MergeTree ORDER BY (t, c1) AS SELECT '2022-09-09 12:00:00', toString(number % 2) FROM numbers(2) UNION ALL SELECT '2022-09-09 12:00:30', toString(number % 2)|| 'x' FROM numbers(3)" -$CLICKHOUSE_CLIENT --optimize_aggregation_in_order=1 -q "SELECT count() FROM - (SELECT toStartOfMinute(t) AS s, c1 FROM t_order_by_monotonic GROUP BY s, c1)" +$CLICKHOUSE_CLIENT "${opts[@]}" --optimize_aggregation_in_order=1 -q "SELECT count() FROM + (SELECT toStartOfMinute(t) AS s, c1 FROM t_order_by_monotonic AS test GROUP BY s, c1)" -$CLICKHOUSE_CLIENT --optimize_read_in_order=1 -q "SELECT toStartOfMinute(t) AS s, c1 FROM t_order_by_monotonic ORDER BY s, c1" +$CLICKHOUSE_CLIENT "${opts[@]}" --optimize_read_in_order=1 -q "SELECT toStartOfMinute(t) AS s, c1 FROM t_order_by_monotonic AS test ORDER BY s, c1" -explain_sort_description "SELECT toStartOfMinute(t) AS s, c1 FROM t_order_by_monotonic ORDER BY s, c1" -explain_sort_description "SELECT toStartOfMinute(t) AS s, c1 FROM t_order_by_monotonic ORDER BY s" +explain_sort_description "SELECT toStartOfMinute(t) AS s, c1 FROM t_order_by_monotonic AS test ORDER BY s, c1" +explain_sort_description "SELECT toStartOfMinute(t) AS s, c1 FROM t_order_by_monotonic AS test ORDER BY s" -$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS t_order_by_monotonic" +$CLICKHOUSE_CLIENT "${opts[@]}" -q "DROP TABLE IF EXISTS t_order_by_monotonic" -$CLICKHOUSE_CLIENT -q "CREATE TABLE t_order_by_monotonic (a Int64, b Int64) ENGINE = MergeTree ORDER BY (a, b)" +$CLICKHOUSE_CLIENT "${opts[@]}" -q "CREATE TABLE t_order_by_monotonic (a Int64, b Int64) ENGINE = MergeTree ORDER BY (a, b)" -$CLICKHOUSE_CLIENT -q "INSERT INTO t_order_by_monotonic VALUES (1, 1) (1, 2), (2, 1) (2, 2)" +$CLICKHOUSE_CLIENT "${opts[@]}" -q "INSERT INTO t_order_by_monotonic VALUES (1, 1) (1, 2), (2, 1) (2, 2)" -explain_sort_description "SELECT * FROM t_order_by_monotonic ORDER BY -a" -explain_sort_description "SELECT * FROM t_order_by_monotonic ORDER BY -a, -b" -explain_sort_description "SELECT * FROM t_order_by_monotonic ORDER BY a DESC, -b" -explain_sort_description "SELECT * FROM t_order_by_monotonic ORDER BY -a, b DESC" -explain_sort_description "SELECT * FROM t_order_by_monotonic ORDER BY -a, b" -explain_sort_description "SELECT * FROM t_order_by_monotonic ORDER BY a, -b" +explain_sort_description "SELECT * FROM t_order_by_monotonic AS test ORDER BY -a" +explain_sort_description "SELECT * FROM t_order_by_monotonic AS test ORDER BY -a, -b" +explain_sort_description "SELECT * FROM t_order_by_monotonic AS test ORDER BY a DESC, -b" +explain_sort_description "SELECT * FROM t_order_by_monotonic AS test ORDER BY -a, b DESC" +explain_sort_description "SELECT * FROM t_order_by_monotonic AS test ORDER BY -a, b" +explain_sort_description "SELECT * FROM t_order_by_monotonic AS test ORDER BY a, -b" -$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS t_order_by_monotonic" +$CLICKHOUSE_CLIENT "${opts[@]}" -q "DROP TABLE IF EXISTS t_order_by_monotonic" diff --git a/tests/queries/0_stateless/02466_distributed_query_profiler.sql b/tests/queries/0_stateless/02466_distributed_query_profiler.sql index 9fc2fe7b4bd..171cc2a7563 100644 --- a/tests/queries/0_stateless/02466_distributed_query_profiler.sql +++ b/tests/queries/0_stateless/02466_distributed_query_profiler.sql @@ -12,10 +12,4 @@ settings -- This is to activate as much signals as possible to trigger EINTR query_profiler_real_time_period_ns=1, -- This is to use MultiplexedConnections - use_hedged_requests=0, - -- This is to make the initiator waiting for cancel packet in MultiplexedConnections::getReplicaForReading() - -- - -- NOTE: that even smaller sleep will be enough to trigger this problem - -- with 100% probability, however just to make it more reliable, increase - -- it to 2 seconds. - sleep_in_receive_cancel_ms=2000; + use_hedged_requests=0; diff --git a/tests/queries/0_stateless/02473_optimize_old_parts.reference b/tests/queries/0_stateless/02473_optimize_old_parts.reference index 9002d73ff27..e80812bddcd 100644 --- a/tests/queries/0_stateless/02473_optimize_old_parts.reference +++ b/tests/queries/0_stateless/02473_optimize_old_parts.reference @@ -8,5 +8,3 @@ With merge replicated any part range 1 With merge replicated partition only 1 -With merge partition only and new parts -3 diff --git a/tests/queries/0_stateless/02473_optimize_old_parts.sh b/tests/queries/0_stateless/02473_optimize_old_parts.sh new file mode 100755 index 00000000000..0ef9a43f833 --- /dev/null +++ b/tests/queries/0_stateless/02473_optimize_old_parts.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash +# Tags: long + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +# Wait for number of parts in table $1 to become $2. +# Print the changed value. If no changes for $3 seconds, prints initial value. +wait_for_number_of_parts() { + for _ in `seq $3` + do + sleep 1 + res=`$CLICKHOUSE_CLIENT -q "SELECT count(*) FROM system.parts WHERE database = currentDatabase() AND table='$1' AND active"` + if [ "$res" -eq "$2" ] + then + echo "$res" + return + fi + done + echo "$res" +} + +$CLICKHOUSE_CLIENT -nmq " +DROP TABLE IF EXISTS test_without_merge; +DROP TABLE IF EXISTS test_with_merge; +DROP TABLE IF EXISTS test_replicated; + +SELECT 'Without merge'; + +CREATE TABLE test_without_merge (i Int64) ENGINE = MergeTree ORDER BY i SETTINGS merge_selecting_sleep_ms=1000; +INSERT INTO test_without_merge SELECT 1; +INSERT INTO test_without_merge SELECT 2; +INSERT INTO test_without_merge SELECT 3;" + +wait_for_number_of_parts 'test_without_merge' 1 10 + +$CLICKHOUSE_CLIENT -nmq " +DROP TABLE test_without_merge; + +SELECT 'With merge any part range'; + +CREATE TABLE test_with_merge (i Int64) ENGINE = MergeTree ORDER BY i +SETTINGS min_age_to_force_merge_seconds=1, merge_selecting_sleep_ms=1000, min_age_to_force_merge_on_partition_only=false; +INSERT INTO test_with_merge SELECT 1; +INSERT INTO test_with_merge SELECT 2; +INSERT INTO test_with_merge SELECT 3;" + +wait_for_number_of_parts 'test_with_merge' 1 100 + +$CLICKHOUSE_CLIENT -nmq " +DROP TABLE test_with_merge; + +SELECT 'With merge partition only'; + +CREATE TABLE test_with_merge (i Int64) ENGINE = MergeTree ORDER BY i +SETTINGS min_age_to_force_merge_seconds=1, merge_selecting_sleep_ms=1000, min_age_to_force_merge_on_partition_only=true; +INSERT INTO test_with_merge SELECT 1; +INSERT INTO test_with_merge SELECT 2; +INSERT INTO test_with_merge SELECT 3;" + +wait_for_number_of_parts 'test_with_merge' 1 100 + +$CLICKHOUSE_CLIENT -nmq " +DROP TABLE test_with_merge; + +SELECT 'With merge replicated any part range'; + +CREATE TABLE test_replicated (i Int64) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{database}/test02473', 'node') ORDER BY i +SETTINGS min_age_to_force_merge_seconds=1, merge_selecting_sleep_ms=1000, min_age_to_force_merge_on_partition_only=false; +INSERT INTO test_replicated SELECT 1; +INSERT INTO test_replicated SELECT 2; +INSERT INTO test_replicated SELECT 3;" + +wait_for_number_of_parts 'test_replicated' 1 100 + +$CLICKHOUSE_CLIENT -nmq " +DROP TABLE test_replicated; + +SELECT 'With merge replicated partition only'; + +CREATE TABLE test_replicated (i Int64) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{database}/test02473_partition_only', 'node') ORDER BY i +SETTINGS min_age_to_force_merge_seconds=1, merge_selecting_sleep_ms=1000, min_age_to_force_merge_on_partition_only=true; +INSERT INTO test_replicated SELECT 1; +INSERT INTO test_replicated SELECT 2; +INSERT INTO test_replicated SELECT 3;" + +wait_for_number_of_parts 'test_replicated' 1 100 + +$CLICKHOUSE_CLIENT -nmq " +DROP TABLE test_replicated;" diff --git a/tests/queries/0_stateless/02473_optimize_old_parts.sql b/tests/queries/0_stateless/02473_optimize_old_parts.sql deleted file mode 100644 index c2bd37033c1..00000000000 --- a/tests/queries/0_stateless/02473_optimize_old_parts.sql +++ /dev/null @@ -1,87 +0,0 @@ --- Tags: long - -DROP TABLE IF EXISTS test_without_merge; -DROP TABLE IF EXISTS test_with_merge; -DROP TABLE IF EXISTS test_replicated; - -SELECT 'Without merge'; - -CREATE TABLE test_without_merge (i Int64) ENGINE = MergeTree ORDER BY i; -INSERT INTO test_without_merge SELECT 1; -INSERT INTO test_without_merge SELECT 2; -INSERT INTO test_without_merge SELECT 3; - -SELECT sleepEachRow(1) FROM numbers(9) FORMAT Null; -SELECT count(*) FROM system.parts WHERE database = currentDatabase() AND table='test_without_merge' AND active; - -DROP TABLE test_without_merge; - -SELECT 'With merge any part range'; - -CREATE TABLE test_with_merge (i Int64) ENGINE = MergeTree ORDER BY i -SETTINGS min_age_to_force_merge_seconds=3, min_age_to_force_merge_on_partition_only=false; -INSERT INTO test_with_merge SELECT 1; -INSERT INTO test_with_merge SELECT 2; -INSERT INTO test_with_merge SELECT 3; - -SELECT sleepEachRow(1) FROM numbers(9) FORMAT Null; -SELECT count(*) FROM system.parts WHERE database = currentDatabase() AND table='test_with_merge' AND active; - -DROP TABLE test_with_merge; - -SELECT 'With merge partition only'; - -CREATE TABLE test_with_merge (i Int64) ENGINE = MergeTree ORDER BY i -SETTINGS min_age_to_force_merge_seconds=3, min_age_to_force_merge_on_partition_only=true; -INSERT INTO test_with_merge SELECT 1; -INSERT INTO test_with_merge SELECT 2; -INSERT INTO test_with_merge SELECT 3; - -SELECT sleepEachRow(1) FROM numbers(9) FORMAT Null; -SELECT count(*) FROM system.parts WHERE database = currentDatabase() AND table='test_with_merge' AND active; - -DROP TABLE test_with_merge; - -SELECT 'With merge replicated any part range'; - -CREATE TABLE test_replicated (i Int64) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{database}/test02473', 'node') ORDER BY i -SETTINGS min_age_to_force_merge_seconds=3, min_age_to_force_merge_on_partition_only=false; -INSERT INTO test_replicated SELECT 1; -INSERT INTO test_replicated SELECT 2; -INSERT INTO test_replicated SELECT 3; - -SELECT sleepEachRow(1) FROM numbers(9) FORMAT Null; -SELECT count(*) FROM system.parts WHERE database = currentDatabase() AND table='test_replicated' AND active; - -DROP TABLE test_replicated; - -SELECT 'With merge replicated partition only'; - -CREATE TABLE test_replicated (i Int64) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{database}/test02473_partition_only', 'node') ORDER BY i -SETTINGS min_age_to_force_merge_seconds=3, min_age_to_force_merge_on_partition_only=true; -INSERT INTO test_replicated SELECT 1; -INSERT INTO test_replicated SELECT 2; -INSERT INTO test_replicated SELECT 3; - -SELECT sleepEachRow(1) FROM numbers(9) FORMAT Null; -SELECT count(*) FROM system.parts WHERE database = currentDatabase() AND table='test_replicated' AND active; - -DROP TABLE test_replicated; - -SELECT 'With merge partition only and new parts'; - -CREATE TABLE test_with_merge (i Int64) ENGINE = MergeTree ORDER BY i -SETTINGS min_age_to_force_merge_seconds=3, min_age_to_force_merge_on_partition_only=true; -SYSTEM STOP MERGES test_with_merge; --- These three parts will have min_age=6 at the time of merge -INSERT INTO test_with_merge SELECT 1; -INSERT INTO test_with_merge SELECT 2; -SELECT sleepEachRow(1) FROM numbers(9) FORMAT Null; --- These three parts will have min_age=0 at the time of merge --- and so, nothing will be merged. -INSERT INTO test_with_merge SELECT 3; -SYSTEM START MERGES test_with_merge; - -SELECT count(*) FROM system.parts WHERE database = currentDatabase() AND table='test_with_merge' AND active; - -DROP TABLE test_with_merge; diff --git a/tests/queries/0_stateless/02477_logical_expressions_optimizer_low_cardinality.reference b/tests/queries/0_stateless/02477_logical_expressions_optimizer_low_cardinality.reference index dcfcac737c3..84589668d64 100644 --- a/tests/queries/0_stateless/02477_logical_expressions_optimizer_low_cardinality.reference +++ b/tests/queries/0_stateless/02477_logical_expressions_optimizer_low_cardinality.reference @@ -1,6 +1,62 @@ SELECT a FROM t_logical_expressions_optimizer_low_cardinality WHERE a IN (\'x\', \'y\') +QUERY id: 0 + PROJECTION COLUMNS + a LowCardinality(String) + PROJECTION + LIST id: 1, nodes: 1 + COLUMN id: 2, column_name: a, result_type: LowCardinality(String), source_id: 3 + JOIN TREE + TABLE id: 3, table_name: default.t_logical_expressions_optimizer_low_cardinality + WHERE + FUNCTION id: 4, function_name: in, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 5, nodes: 2 + COLUMN id: 2, column_name: a, result_type: LowCardinality(String), source_id: 3 + CONSTANT id: 6, constant_value: Tuple_(\'x\', \'y\'), constant_value_type: Tuple(String, String) + SETTINGS allow_experimental_analyzer=1 +SELECT a +FROM t_logical_expressions_optimizer_low_cardinality +WHERE (a = \'x\') OR (\'y\' = a) +QUERY id: 0 + PROJECTION COLUMNS + a LowCardinality(String) + PROJECTION + LIST id: 1, nodes: 1 + COLUMN id: 2, column_name: a, result_type: LowCardinality(String), source_id: 3 + JOIN TREE + TABLE id: 3, table_name: default.t_logical_expressions_optimizer_low_cardinality + WHERE + FUNCTION id: 4, function_name: in, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 5, nodes: 2 + COLUMN id: 2, column_name: a, result_type: LowCardinality(String), source_id: 3 + CONSTANT id: 6, constant_value: Tuple_(\'x\', \'y\'), constant_value_type: Tuple(String, String) + SETTINGS allow_experimental_analyzer=1 SELECT a FROM t_logical_expressions_optimizer_low_cardinality WHERE (b = 0) OR (b = 1) +QUERY id: 0 + PROJECTION COLUMNS + a LowCardinality(String) + PROJECTION + LIST id: 1, nodes: 1 + COLUMN id: 2, column_name: a, result_type: LowCardinality(String), source_id: 3 + JOIN TREE + TABLE id: 3, table_name: default.t_logical_expressions_optimizer_low_cardinality + WHERE + FUNCTION id: 4, function_name: or, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 5, nodes: 2 + FUNCTION id: 6, function_name: equals, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 7, nodes: 2 + COLUMN id: 8, column_name: b, result_type: UInt32, source_id: 3 + CONSTANT id: 9, constant_value: UInt64_0, constant_value_type: UInt8 + FUNCTION id: 10, function_name: equals, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 11, nodes: 2 + COLUMN id: 8, column_name: b, result_type: UInt32, source_id: 3 + CONSTANT id: 12, constant_value: UInt64_1, constant_value_type: UInt8 + SETTINGS allow_experimental_analyzer=1 diff --git a/tests/queries/0_stateless/02477_logical_expressions_optimizer_low_cardinality.sql b/tests/queries/0_stateless/02477_logical_expressions_optimizer_low_cardinality.sql index be355a05675..14f8ad830e7 100644 --- a/tests/queries/0_stateless/02477_logical_expressions_optimizer_low_cardinality.sql +++ b/tests/queries/0_stateless/02477_logical_expressions_optimizer_low_cardinality.sql @@ -4,7 +4,11 @@ CREATE TABLE t_logical_expressions_optimizer_low_cardinality (a LowCardinality(S -- LowCardinality case, ignore optimize_min_equality_disjunction_chain_length limit, optimzer applied EXPLAIN SYNTAX SELECT a FROM t_logical_expressions_optimizer_low_cardinality WHERE a = 'x' OR a = 'y'; +EXPLAIN QUERY TREE SELECT a FROM t_logical_expressions_optimizer_low_cardinality WHERE a = 'x' OR a = 'y' SETTINGS allow_experimental_analyzer = 1; +EXPLAIN SYNTAX SELECT a FROM t_logical_expressions_optimizer_low_cardinality WHERE a = 'x' OR 'y' = a; +EXPLAIN QUERY TREE SELECT a FROM t_logical_expressions_optimizer_low_cardinality WHERE a = 'x' OR 'y' = a SETTINGS allow_experimental_analyzer = 1; -- Non-LowCardinality case, optimizer not applied for short chains EXPLAIN SYNTAX SELECT a FROM t_logical_expressions_optimizer_low_cardinality WHERE b = 0 OR b = 1; +EXPLAIN QUERY TREE SELECT a FROM t_logical_expressions_optimizer_low_cardinality WHERE b = 0 OR b = 1 SETTINGS allow_experimental_analyzer = 1; DROP TABLE t_logical_expressions_optimizer_low_cardinality; diff --git a/tests/queries/0_stateless/02480_tlp_nan.reference b/tests/queries/0_stateless/02480_tlp_nan.reference index ea4aa44fa89..befd1f66564 100644 --- a/tests/queries/0_stateless/02480_tlp_nan.reference +++ b/tests/queries/0_stateless/02480_tlp_nan.reference @@ -1,10 +1,21 @@ +-- {echo} +SELECT sqrt(-1) as x, not(x), not(not(x)), (not(x)) IS NULL SETTINGS allow_experimental_analyzer=1; nan 0 1 0 +SELECT sqrt(-1) as x, not(x), not(not(x)), (not(x)) IS NULL SETTINGS allow_experimental_analyzer=0; nan 0 1 0 +SELECT -inf as x, not(x), not(not(x)), (not(x)) IS NULL SETTINGS allow_experimental_analyzer=1; -inf 0 1 0 +SELECT -inf as x, not(x), not(not(x)), (not(x)) IS NULL SETTINGS allow_experimental_analyzer=0; -inf 0 1 0 +SELECT NULL as x, not(x), not(not(x)), (not(x)) IS NULL SETTINGS allow_experimental_analyzer=1; \N \N \N 1 +SELECT NULL as x, not(x), not(not(x)), (not(x)) IS NULL SETTINGS allow_experimental_analyzer=0; \N \N \N 1 +SELECT inf as x, not(x), not(not(x)), (not(x)) IS NULL SETTINGS allow_experimental_analyzer=1; inf 0 1 0 +SELECT inf as x, not(x), not(not(x)), (not(x)) IS NULL SETTINGS allow_experimental_analyzer=0; inf 0 1 0 +SELECT nan as x, not(x), not(not(x)), (not(x)) IS NULL SETTINGS allow_experimental_analyzer=1; nan 0 1 0 +SELECT nan as x, not(x), not(not(x)), (not(x)) IS NULL SETTINGS allow_experimental_analyzer=0; nan 0 1 0 diff --git a/tests/queries/0_stateless/02483_cuturlparameter_with_arrays.reference b/tests/queries/0_stateless/02483_cuturlparameter_with_arrays.reference index dd677873c7c..348408a15cc 100644 --- a/tests/queries/0_stateless/02483_cuturlparameter_with_arrays.reference +++ b/tests/queries/0_stateless/02483_cuturlparameter_with_arrays.reference @@ -1,4 +1,5 @@ -- { echoOn } + SELECT cutURLParameter('http://bigmir.net/?a=b&c=d', []), cutURLParameter('http://bigmir.net/?a=b&c=d', ['a']), @@ -30,7 +31,7 @@ SELECT FORMAT Vertical; Row 1: ────── -cutURLParameter('http://bigmir.net/?a=b&c=d', []): http://bigmir.net/?a=b&c=d +cutURLParameter('http://bigmir.net/?a=b&c=d', array()): http://bigmir.net/?a=b&c=d cutURLParameter('http://bigmir.net/?a=b&c=d', ['a']): http://bigmir.net/?c=d cutURLParameter('http://bigmir.net/?a=b&c=d', ['a', 'c']): http://bigmir.net/? cutURLParameter('http://bigmir.net/?a=b&c=d', ['c']): http://bigmir.net/?a=b @@ -43,7 +44,7 @@ cutURLParameter('http://bigmir.net/?a=b&c=d#e&g=h', ['c', 'g']): http: cutURLParameter('http://bigmir.net/?a=b&c=d#e&g=h', ['e', 'g']): http://bigmir.net/?a=b&c=d#e cutURLParameter('http://bigmir.net/?a=b&c=d#test?e=f&g=h', ['test', 'e']): http://bigmir.net/?a=b&c=d#test?g=h cutURLParameter('http://bigmir.net/?a=b&c=d#test?e=f&g=h', ['test', 'g']): http://bigmir.net/?a=b&c=d#test?e=f -cutURLParameter('//bigmir.net/?a=b&c=d', []): //bigmir.net/?a=b&c=d +cutURLParameter('//bigmir.net/?a=b&c=d', array()): //bigmir.net/?a=b&c=d cutURLParameter('//bigmir.net/?a=b&c=d', ['a']): //bigmir.net/?c=d cutURLParameter('//bigmir.net/?a=b&c=d', ['a', 'c']): //bigmir.net/? cutURLParameter('//bigmir.net/?a=b&c=d#e=f', ['a', 'e']): //bigmir.net/?c=d# @@ -88,7 +89,7 @@ SELECT FORMAT Vertical; Row 1: ────── -cutURLParameter(materialize('http://bigmir.net/?a=b&c=d'), []): http://bigmir.net/?a=b&c=d +cutURLParameter(materialize('http://bigmir.net/?a=b&c=d'), array()): http://bigmir.net/?a=b&c=d cutURLParameter(materialize('http://bigmir.net/?a=b&c=d'), ['a']): http://bigmir.net/?c=d cutURLParameter(materialize('http://bigmir.net/?a=b&c=d'), ['a', 'c']): http://bigmir.net/? cutURLParameter(materialize('http://bigmir.net/?a=b&c=d'), ['c']): http://bigmir.net/?a=b @@ -101,7 +102,7 @@ cutURLParameter(materialize('http://bigmir.net/?a=b&c=d#e&g=h'), ['c', 'g']): cutURLParameter(materialize('http://bigmir.net/?a=b&c=d#e&g=h'), ['e', 'g']): http://bigmir.net/?a=b&c=d#e cutURLParameter(materialize('http://bigmir.net/?a=b&c=d#test?e=f&g=h'), ['test', 'e']): http://bigmir.net/?a=b&c=d#test?g=h cutURLParameter(materialize('http://bigmir.net/?a=b&c=d#test?e=f&g=h'), ['test', 'g']): http://bigmir.net/?a=b&c=d#test?e=f -cutURLParameter(materialize('//bigmir.net/?a=b&c=d'), []): //bigmir.net/?a=b&c=d +cutURLParameter(materialize('//bigmir.net/?a=b&c=d'), array()): //bigmir.net/?a=b&c=d cutURLParameter(materialize('//bigmir.net/?a=b&c=d'), ['a']): //bigmir.net/?c=d cutURLParameter(materialize('//bigmir.net/?a=b&c=d'), ['a', 'c']): //bigmir.net/? cutURLParameter(materialize('//bigmir.net/?a=b&c=d#e=f'), ['a', 'e']): //bigmir.net/?c=d# diff --git a/tests/queries/0_stateless/02483_cuturlparameter_with_arrays.sql b/tests/queries/0_stateless/02483_cuturlparameter_with_arrays.sql index ea2d6ae104f..6d64d2685b7 100644 --- a/tests/queries/0_stateless/02483_cuturlparameter_with_arrays.sql +++ b/tests/queries/0_stateless/02483_cuturlparameter_with_arrays.sql @@ -1,4 +1,7 @@ +SET allow_experimental_analyzer = 1; + -- { echoOn } + SELECT cutURLParameter('http://bigmir.net/?a=b&c=d', []), cutURLParameter('http://bigmir.net/?a=b&c=d', ['a']), diff --git a/tests/queries/0_stateless/02494_query_cache_explain.reference b/tests/queries/0_stateless/02494_query_cache_explain.reference index ecc965ac391..690e75bca7c 100644 --- a/tests/queries/0_stateless/02494_query_cache_explain.reference +++ b/tests/queries/0_stateless/02494_query_cache_explain.reference @@ -1,9 +1,9 @@ 1 1 -Expression ((Projection + Before ORDER BY)) +Expression ((Project names + (Projection + Change column names to column identifiers))) Limit (preliminary LIMIT (without OFFSET)) ReadFromStorage (SystemNumbers) -Expression ((Projection + Before ORDER BY)) +Expression ((Project names + (Projection + Change column names to column identifiers))) Limit (preliminary LIMIT (without OFFSET)) ReadFromStorage (SystemNumbers) (Expression) diff --git a/tests/queries/0_stateless/02494_query_cache_explain.sql b/tests/queries/0_stateless/02494_query_cache_explain.sql index 67717efde13..68b7e0005f8 100644 --- a/tests/queries/0_stateless/02494_query_cache_explain.sql +++ b/tests/queries/0_stateless/02494_query_cache_explain.sql @@ -1,6 +1,7 @@ -- Tags: no-parallel -- Tag no-parallel: Messes with internal cache +SET allow_experimental_analyzer = 1; SET allow_experimental_query_cache = true; SYSTEM DROP QUERY CACHE; diff --git a/tests/queries/0_stateless/02500_remove_redundant_distinct.reference b/tests/queries/0_stateless/02500_remove_redundant_distinct.reference index 32ddab4886c..2e049dbc936 100644 --- a/tests/queries/0_stateless/02500_remove_redundant_distinct.reference +++ b/tests/queries/0_stateless/02500_remove_redundant_distinct.reference @@ -464,3 +464,16 @@ Expression ((Projection + (Before ORDER BY + (Projection + Before ORDER BY)))) 1 0 +-- DISTINCT COUNT() with GROUP BY => do _not_ remove DISTINCT +-- query +select distinct count() from numbers(10) group by number +-- explain +Expression (Projection) + Distinct + Distinct (Preliminary DISTINCT) + Expression (Before ORDER BY) + Aggregating + Expression (Before GROUP BY) + ReadFromStorage (SystemNumbers) +-- execute +1 diff --git a/tests/queries/0_stateless/02500_remove_redundant_distinct.sh b/tests/queries/0_stateless/02500_remove_redundant_distinct.sh index 879cc776fe1..d550b057853 100755 --- a/tests/queries/0_stateless/02500_remove_redundant_distinct.sh +++ b/tests/queries/0_stateless/02500_remove_redundant_distinct.sh @@ -256,3 +256,7 @@ FROM GROUP BY a WITH TOTALS )" run_query "$query" + +echo "-- DISTINCT COUNT() with GROUP BY => do _not_ remove DISTINCT" +query="select distinct count() from numbers(10) group by number" +run_query "$query" diff --git a/tests/queries/0_stateless/02514_analyzer_drop_join_on.reference b/tests/queries/0_stateless/02514_analyzer_drop_join_on.reference index af6f07e3f95..8b4cafc3260 100644 --- a/tests/queries/0_stateless/02514_analyzer_drop_join_on.reference +++ b/tests/queries/0_stateless/02514_analyzer_drop_join_on.reference @@ -30,20 +30,20 @@ Header: count() UInt64 Expression ((JOIN actions + Change column names to column identifiers)) Header: default.a.a1_1 UInt64 default.a.a2_4 String - ReadFromStorage (Memory) + ReadFromMemoryStorage Header: a1 UInt64 a2 String Expression ((JOIN actions + Change column names to column identifiers)) Header: default.b.b1_0 UInt64 - ReadFromStorage (Memory) + ReadFromMemoryStorage Header: b1 UInt64 Expression ((JOIN actions + Change column names to column identifiers)) Header: default.c.c1_2 UInt64 - ReadFromStorage (Memory) + ReadFromMemoryStorage Header: c1 UInt64 Expression ((JOIN actions + Change column names to column identifiers)) Header: default.d.d1_3 UInt64 - ReadFromStorage (Memory) + ReadFromMemoryStorage Header: d1 UInt64 EXPLAIN PLAN header = 1 SELECT a.a2, d.d2 FROM a JOIN b USING (k) JOIN c USING (k) JOIN d USING (k) @@ -70,21 +70,21 @@ Header: a2 String Expression (Change column names to column identifiers) Header: default.a.a2_0 String default.a.k_2 UInt64 - ReadFromStorage (Memory) + ReadFromMemoryStorage Header: a2 String k UInt64 Expression (Change column names to column identifiers) Header: default.b.k_3 UInt64 - ReadFromStorage (Memory) + ReadFromMemoryStorage Header: k UInt64 Expression (Change column names to column identifiers) Header: default.c.k_4 UInt64 - ReadFromStorage (Memory) + ReadFromMemoryStorage Header: k UInt64 Expression (Change column names to column identifiers) Header: default.d.d2_1 String default.d.k_5 UInt64 - ReadFromStorage (Memory) + ReadFromMemoryStorage Header: d2 String k UInt64 EXPLAIN PLAN header = 1 @@ -130,19 +130,19 @@ Header: bx String Expression ((JOIN actions + Change column names to column identifiers)) Header: default.a.a1_2 UInt64 default.a.a2_6 String - ReadFromStorage (Memory) + ReadFromMemoryStorage Header: a1 UInt64 a2 String Expression ((JOIN actions + (Change column names to column identifiers + (Project names + (Projection + Change column names to column identifiers))))) Header: b.b1_1 UInt64 b.bx_0 String - ReadFromStorage (Memory) + ReadFromMemoryStorage Header: b1 UInt64 b2 String Expression ((JOIN actions + Change column names to column identifiers)) Header: default.c.c1_3 UInt64 default.c.c2_5 String - ReadFromStorage (Memory) + ReadFromMemoryStorage Header: c1 UInt64 c2 String Expression ((JOIN actions + (Change column names to column identifiers + (Project names + (Projection + Change column names to column identifiers))))) diff --git a/tests/queries/0_stateless/02521_cannot-find-column-in-projection.sql b/tests/queries/0_stateless/02521_cannot-find-column-in-projection.sql deleted file mode 100644 index 31602c5bae2..00000000000 --- a/tests/queries/0_stateless/02521_cannot-find-column-in-projection.sql +++ /dev/null @@ -1,3 +0,0 @@ -create table test(day Date, id UInt32) engine=MergeTree partition by day order by tuple(); -insert into test select toDate('2023-01-05') AS day, number from numbers(10); -with toUInt64(id) as id_with select day, count(id_with) from test where day >= '2023-01-01' group by day limit 1000; -- { serverError NOT_FOUND_COLUMN_IN_BLOCK } diff --git a/tests/queries/0_stateless/02521_cannot_find_column_in_projection.reference b/tests/queries/0_stateless/02521_cannot_find_column_in_projection.reference new file mode 100644 index 00000000000..2cd767c8054 --- /dev/null +++ b/tests/queries/0_stateless/02521_cannot_find_column_in_projection.reference @@ -0,0 +1 @@ +2023-01-05 10 diff --git a/tests/queries/0_stateless/02521_cannot_find_column_in_projection.sql b/tests/queries/0_stateless/02521_cannot_find_column_in_projection.sql new file mode 100644 index 00000000000..255c6f56ab3 --- /dev/null +++ b/tests/queries/0_stateless/02521_cannot_find_column_in_projection.sql @@ -0,0 +1,7 @@ +SET allow_experimental_analyzer = 1; + +drop table if exists test; +create table test(day Date, id UInt32) engine=MergeTree partition by day order by tuple(); +insert into test select toDate('2023-01-05') AS day, number from numbers(10); +with toUInt64(id) as id_with select day, count(id_with) from test where day >= '2023-01-01' group by day limit 1000; +drop table test; diff --git a/tests/queries/0_stateless/02525_different_engines_in_temporary_tables.reference b/tests/queries/0_stateless/02525_different_engines_in_temporary_tables.reference new file mode 100644 index 00000000000..3d1916b29f6 --- /dev/null +++ b/tests/queries/0_stateless/02525_different_engines_in_temporary_tables.reference @@ -0,0 +1,14 @@ +1 a +2 b +3 c +0 +0 +1 a +2 b +3 c +1 a +2 b +3 c +1 a +2 b +3 c diff --git a/tests/queries/0_stateless/02525_different_engines_in_temporary_tables.sql b/tests/queries/0_stateless/02525_different_engines_in_temporary_tables.sql new file mode 100644 index 00000000000..7ebc05dfece --- /dev/null +++ b/tests/queries/0_stateless/02525_different_engines_in_temporary_tables.sql @@ -0,0 +1,66 @@ +DROP TEMPORARY TABLE IF EXISTS table_merge_tree_02525; +CREATE TEMPORARY TABLE table_merge_tree_02525 +( + id UInt64, + info String +) +ENGINE = MergeTree +ORDER BY id +PRIMARY KEY id; +INSERT INTO table_merge_tree_02525 VALUES (1, 'a'), (2, 'b'), (3, 'c'); +SELECT * FROM table_merge_tree_02525; +-- Check that temporary table with MergeTree is not sent to remote servers +-- The query with remote() should not fail +SELECT dummy FROM remote('127.0.0.{1,2}', system, one); +DROP TEMPORARY TABLE table_merge_tree_02525; + +DROP TEMPORARY TABLE IF EXISTS table_log_02525; +CREATE TEMPORARY TABLE table_log_02525 +( + id UInt64, + info String +) +ENGINE = Log; +INSERT INTO table_log_02525 VALUES (1, 'a'), (2, 'b'), (3, 'c'); +SELECT * FROM table_log_02525; +DROP TEMPORARY TABLE table_log_02525; + +DROP TEMPORARY TABLE IF EXISTS table_stripe_log_02525; +CREATE TEMPORARY TABLE table_stripe_log_02525 +( + id UInt64, + info String +) +ENGINE = StripeLog; +INSERT INTO table_stripe_log_02525 VALUES (1, 'a'), (2, 'b'), (3, 'c'); +SELECT * FROM table_stripe_log_02525; +DROP TEMPORARY TABLE table_stripe_log_02525; + +DROP TEMPORARY TABLE IF EXISTS table_tiny_log_02525; +CREATE TEMPORARY TABLE table_tiny_log_02525 +( + id UInt64, + info String +) +ENGINE = TinyLog; +INSERT INTO table_tiny_log_02525 VALUES (1, 'a'), (2, 'b'), (3, 'c'); +SELECT * FROM table_tiny_log_02525; +DROP TEMPORARY TABLE table_tiny_log_02525; + +DROP TEMPORARY TABLE IF EXISTS table_replicated_merge_tree_02525; +CREATE TEMPORARY TABLE table_replicated_merge_tree_02525 +( + id UInt64, + info String +) +ENGINE ReplicatedMergeTree('/clickhouse/tables/{database}/test_02525/table_replicated_merge_tree_02525', 'r1') +ORDER BY id +PRIMARY KEY id; -- { serverError INCORRECT_QUERY } + +DROP TEMPORARY TABLE IF EXISTS table_keeper_map_02525; +CREATE TEMPORARY TABLE table_keeper_map_02525 +( + key String, + value UInt32 +) Engine=KeeperMap('/' || currentDatabase() || '/test02525') +PRIMARY KEY(key); -- { serverError INCORRECT_QUERY } diff --git a/tests/queries/0_stateless/02534_keyed_siphash.reference b/tests/queries/0_stateless/02534_keyed_siphash.reference index 52e92f37720..3606b9a41db 100644 --- a/tests/queries/0_stateless/02534_keyed_siphash.reference +++ b/tests/queries/0_stateless/02534_keyed_siphash.reference @@ -191,6 +191,6 @@ E51B38608EF25F57 1 1 E28DBDE7FE22E41C -1CE422FEE7BD8DE20000000000000000 +1 E28DBDE7FE22E41C -1CE422FEE7BD8DE20000000000000000 +1 diff --git a/tests/queries/0_stateless/02534_keyed_siphash.sql b/tests/queries/0_stateless/02534_keyed_siphash.sql index 3c41efd7d58..9c914f586f0 100644 --- a/tests/queries/0_stateless/02534_keyed_siphash.sql +++ b/tests/queries/0_stateless/02534_keyed_siphash.sql @@ -269,6 +269,6 @@ select sipHash64Keyed(toUInt64(0), '1'); -- { serverError 48 } select sipHash128Keyed(toUInt64(0), '1'); -- { serverError 48 } select hex(sipHash64()); -select hex(sipHash128()); +SELECT hex(sipHash128()) = hex(reverse(unhex('1CE422FEE7BD8DE20000000000000000'))) or hex(sipHash128()) = '1CE422FEE7BD8DE20000000000000000'; select hex(sipHash64Keyed()); -select hex(sipHash128Keyed()); +SELECT hex(sipHash128Keyed()) = hex(reverse(unhex('1CE422FEE7BD8DE20000000000000000'))) or hex(sipHash128Keyed()) = '1CE422FEE7BD8DE20000000000000000'; diff --git a/tests/queries/0_stateless/02535_analyzer_group_by_use_nulls.reference b/tests/queries/0_stateless/02535_analyzer_group_by_use_nulls.reference new file mode 100644 index 00000000000..50755627996 --- /dev/null +++ b/tests/queries/0_stateless/02535_analyzer_group_by_use_nulls.reference @@ -0,0 +1,256 @@ +-- { echoOn } +SELECT number, number % 2, sum(number) AS val +FROM numbers(10) +GROUP BY ROLLUP(number, number % 2) +ORDER BY (number, number % 2, val) +SETTINGS group_by_use_nulls=1; +0 0 0 +1 1 1 +2 0 2 +3 1 3 +4 0 4 +5 1 5 +6 0 6 +7 1 7 +8 0 8 +9 1 9 +\N \N 45 +set optimize_group_by_function_keys = 0; +SELECT number, number % 2, sum(number) AS val +FROM numbers(10) +GROUP BY ROLLUP(number, number % 2) +ORDER BY (number, number % 2, val) +SETTINGS group_by_use_nulls=1; +0 0 0 +0 \N 0 +1 1 1 +1 \N 1 +2 0 2 +2 \N 2 +3 1 3 +3 \N 3 +4 0 4 +4 \N 4 +5 1 5 +5 \N 5 +6 0 6 +6 \N 6 +7 1 7 +7 \N 7 +8 0 8 +8 \N 8 +9 1 9 +9 \N 9 +\N \N 45 +SELECT number, number % 2, sum(number) AS val +FROM numbers(10) +GROUP BY ROLLUP(number, number % 2) +ORDER BY (number, number % 2, val) +SETTINGS group_by_use_nulls=0; +0 0 0 +0 0 0 +0 0 45 +1 0 1 +1 1 1 +2 0 2 +2 0 2 +3 0 3 +3 1 3 +4 0 4 +4 0 4 +5 0 5 +5 1 5 +6 0 6 +6 0 6 +7 0 7 +7 1 7 +8 0 8 +8 0 8 +9 0 9 +9 1 9 +SELECT number, number % 2, sum(number) AS val +FROM numbers(10) +GROUP BY CUBE(number, number % 2) +ORDER BY (number, number % 2, val) +SETTINGS group_by_use_nulls=1; +0 0 0 +0 \N 0 +1 1 1 +1 \N 1 +2 0 2 +2 \N 2 +3 1 3 +3 \N 3 +4 0 4 +4 \N 4 +5 1 5 +5 \N 5 +6 0 6 +6 \N 6 +7 1 7 +7 \N 7 +8 0 8 +8 \N 8 +9 1 9 +9 \N 9 +\N 0 20 +\N 1 25 +\N \N 45 +SELECT number, number % 2, sum(number) AS val +FROM numbers(10) +GROUP BY CUBE(number, number % 2) +ORDER BY (number, number % 2, val) +SETTINGS group_by_use_nulls=0; +0 0 0 +0 0 0 +0 0 20 +0 0 45 +0 1 25 +1 0 1 +1 1 1 +2 0 2 +2 0 2 +3 0 3 +3 1 3 +4 0 4 +4 0 4 +5 0 5 +5 1 5 +6 0 6 +6 0 6 +7 0 7 +7 1 7 +8 0 8 +8 0 8 +9 0 9 +9 1 9 +SELECT + number, + number % 2, + sum(number) AS val +FROM numbers(10) +GROUP BY + GROUPING SETS ( + (number), + (number % 2) + ) +ORDER BY (number, number % 2, val) +SETTINGS group_by_use_nulls = 1; +0 \N 0 +1 \N 1 +2 \N 2 +3 \N 3 +4 \N 4 +5 \N 5 +6 \N 6 +7 \N 7 +8 \N 8 +9 \N 9 +\N 0 20 +\N 1 25 +SELECT + number, + number % 2, + sum(number) AS val +FROM numbers(10) +GROUP BY + GROUPING SETS ( + (number), + (number % 2) + ) +ORDER BY (number, number % 2, val) +SETTINGS group_by_use_nulls = 0; +0 0 0 +0 0 20 +0 1 25 +1 0 1 +2 0 2 +3 0 3 +4 0 4 +5 0 5 +6 0 6 +7 0 7 +8 0 8 +9 0 9 +SELECT number, number % 2, sum(number) AS val +FROM numbers(10) +GROUP BY ROLLUP(number, number % 2) WITH TOTALS +ORDER BY (number, number % 2, val) +SETTINGS group_by_use_nulls=1; +0 0 0 +0 \N 0 +1 1 1 +1 \N 1 +2 0 2 +2 \N 2 +3 1 3 +3 \N 3 +4 0 4 +4 \N 4 +5 1 5 +5 \N 5 +6 0 6 +6 \N 6 +7 1 7 +7 \N 7 +8 0 8 +8 \N 8 +9 1 9 +9 \N 9 +\N \N 45 + +0 0 45 +SELECT number, number % 2, sum(number) AS val +FROM numbers(10) +GROUP BY CUBE(number, number % 2) WITH TOTALS +ORDER BY (number, number % 2, val) +SETTINGS group_by_use_nulls=1; +0 0 0 +0 \N 0 +1 1 1 +1 \N 1 +2 0 2 +2 \N 2 +3 1 3 +3 \N 3 +4 0 4 +4 \N 4 +5 1 5 +5 \N 5 +6 0 6 +6 \N 6 +7 1 7 +7 \N 7 +8 0 8 +8 \N 8 +9 1 9 +9 \N 9 +\N 0 20 +\N 1 25 +\N \N 45 + +0 0 45 +SELECT + number, + number % 2, + sum(number) AS val +FROM numbers(10) +GROUP BY + GROUPING SETS ( + (number), + (number % 2) + ) +ORDER BY 1, tuple(val) +SETTINGS group_by_use_nulls = 1, max_bytes_before_external_sort=10; +0 \N 0 +1 \N 1 +2 \N 2 +3 \N 3 +4 \N 4 +5 \N 5 +6 \N 6 +7 \N 7 +8 \N 8 +9 \N 9 +\N 0 20 +\N 1 25 diff --git a/tests/queries/0_stateless/02535_analyzer_group_by_use_nulls.sql b/tests/queries/0_stateless/02535_analyzer_group_by_use_nulls.sql new file mode 100644 index 00000000000..a4d4f2f8bc9 --- /dev/null +++ b/tests/queries/0_stateless/02535_analyzer_group_by_use_nulls.sql @@ -0,0 +1,85 @@ +SET allow_experimental_analyzer=1; + +-- { echoOn } +SELECT number, number % 2, sum(number) AS val +FROM numbers(10) +GROUP BY ROLLUP(number, number % 2) +ORDER BY (number, number % 2, val) +SETTINGS group_by_use_nulls=1; + +set optimize_group_by_function_keys = 0; + +SELECT number, number % 2, sum(number) AS val +FROM numbers(10) +GROUP BY ROLLUP(number, number % 2) +ORDER BY (number, number % 2, val) +SETTINGS group_by_use_nulls=1; + +SELECT number, number % 2, sum(number) AS val +FROM numbers(10) +GROUP BY ROLLUP(number, number % 2) +ORDER BY (number, number % 2, val) +SETTINGS group_by_use_nulls=0; + +SELECT number, number % 2, sum(number) AS val +FROM numbers(10) +GROUP BY CUBE(number, number % 2) +ORDER BY (number, number % 2, val) +SETTINGS group_by_use_nulls=1; + +SELECT number, number % 2, sum(number) AS val +FROM numbers(10) +GROUP BY CUBE(number, number % 2) +ORDER BY (number, number % 2, val) +SETTINGS group_by_use_nulls=0; + +SELECT + number, + number % 2, + sum(number) AS val +FROM numbers(10) +GROUP BY + GROUPING SETS ( + (number), + (number % 2) + ) +ORDER BY (number, number % 2, val) +SETTINGS group_by_use_nulls = 1; + +SELECT + number, + number % 2, + sum(number) AS val +FROM numbers(10) +GROUP BY + GROUPING SETS ( + (number), + (number % 2) + ) +ORDER BY (number, number % 2, val) +SETTINGS group_by_use_nulls = 0; + +SELECT number, number % 2, sum(number) AS val +FROM numbers(10) +GROUP BY ROLLUP(number, number % 2) WITH TOTALS +ORDER BY (number, number % 2, val) +SETTINGS group_by_use_nulls=1; + +SELECT number, number % 2, sum(number) AS val +FROM numbers(10) +GROUP BY CUBE(number, number % 2) WITH TOTALS +ORDER BY (number, number % 2, val) +SETTINGS group_by_use_nulls=1; + +SELECT + number, + number % 2, + sum(number) AS val +FROM numbers(10) +GROUP BY + GROUPING SETS ( + (number), + (number % 2) + ) +ORDER BY 1, tuple(val) +SETTINGS group_by_use_nulls = 1, max_bytes_before_external_sort=10; diff --git a/tests/queries/0_stateless/02535_max_parallel_replicas_custom_key.reference b/tests/queries/0_stateless/02535_max_parallel_replicas_custom_key.reference new file mode 100644 index 00000000000..8d0f56ba185 --- /dev/null +++ b/tests/queries/0_stateless/02535_max_parallel_replicas_custom_key.reference @@ -0,0 +1,173 @@ +query='SELECT * FROM cluster(test_cluster_one_shard_three_replicas_localhost, currentDatabase(), 02535_custom_key)' with custom_key='sipHash64(x)' +filter_type='default' max_replicas=1 prefer_localhost_replica=0 +Hello +filter_type='default' max_replicas=2 prefer_localhost_replica=0 +Hello +filter_type='default' max_replicas=3 prefer_localhost_replica=0 +Hello +filter_type='range' max_replicas=1 prefer_localhost_replica=0 +Hello +filter_type='range' max_replicas=2 prefer_localhost_replica=0 +Hello +filter_type='range' max_replicas=3 prefer_localhost_replica=0 +Hello +filter_type='default' max_replicas=1 prefer_localhost_replica=1 +Hello +filter_type='default' max_replicas=2 prefer_localhost_replica=1 +Hello +filter_type='default' max_replicas=3 prefer_localhost_replica=1 +Hello +filter_type='range' max_replicas=1 prefer_localhost_replica=1 +Hello +filter_type='range' max_replicas=2 prefer_localhost_replica=1 +Hello +filter_type='range' max_replicas=3 prefer_localhost_replica=1 +Hello +query='SELECT y, count() FROM cluster(test_cluster_one_shard_three_replicas_localhost, currentDatabase(), 02535_custom_key) GROUP BY y ORDER BY y' with custom_key='y' +filter_type='default' max_replicas=1 prefer_localhost_replica=0 +0 334 +1 333 +2 333 +filter_type='default' max_replicas=2 prefer_localhost_replica=0 +0 334 +1 333 +2 333 +filter_type='default' max_replicas=3 prefer_localhost_replica=0 +0 334 +1 333 +2 333 +filter_type='range' max_replicas=1 prefer_localhost_replica=0 +0 334 +1 333 +2 333 +filter_type='range' max_replicas=2 prefer_localhost_replica=0 +0 334 +1 333 +2 333 +filter_type='range' max_replicas=3 prefer_localhost_replica=0 +0 334 +1 333 +2 333 +filter_type='default' max_replicas=1 prefer_localhost_replica=1 +0 334 +1 333 +2 333 +filter_type='default' max_replicas=2 prefer_localhost_replica=1 +0 334 +1 333 +2 333 +filter_type='default' max_replicas=3 prefer_localhost_replica=1 +0 334 +1 333 +2 333 +filter_type='range' max_replicas=1 prefer_localhost_replica=1 +0 334 +1 333 +2 333 +filter_type='range' max_replicas=2 prefer_localhost_replica=1 +0 334 +1 333 +2 333 +filter_type='range' max_replicas=3 prefer_localhost_replica=1 +0 334 +1 333 +2 333 +query='SELECT y, count() FROM cluster(test_cluster_one_shard_three_replicas_localhost, currentDatabase(), 02535_custom_key) GROUP BY y ORDER BY y' with custom_key='cityHash64(y)' +filter_type='default' max_replicas=1 prefer_localhost_replica=0 +0 334 +1 333 +2 333 +filter_type='default' max_replicas=2 prefer_localhost_replica=0 +0 334 +1 333 +2 333 +filter_type='default' max_replicas=3 prefer_localhost_replica=0 +0 334 +1 333 +2 333 +filter_type='range' max_replicas=1 prefer_localhost_replica=0 +0 334 +1 333 +2 333 +filter_type='range' max_replicas=2 prefer_localhost_replica=0 +0 334 +1 333 +2 333 +filter_type='range' max_replicas=3 prefer_localhost_replica=0 +0 334 +1 333 +2 333 +filter_type='default' max_replicas=1 prefer_localhost_replica=1 +0 334 +1 333 +2 333 +filter_type='default' max_replicas=2 prefer_localhost_replica=1 +0 334 +1 333 +2 333 +filter_type='default' max_replicas=3 prefer_localhost_replica=1 +0 334 +1 333 +2 333 +filter_type='range' max_replicas=1 prefer_localhost_replica=1 +0 334 +1 333 +2 333 +filter_type='range' max_replicas=2 prefer_localhost_replica=1 +0 334 +1 333 +2 333 +filter_type='range' max_replicas=3 prefer_localhost_replica=1 +0 334 +1 333 +2 333 +query='SELECT y, count() FROM cluster(test_cluster_one_shard_three_replicas_localhost, currentDatabase(), 02535_custom_key) GROUP BY y ORDER BY y' with custom_key='cityHash64(y) + 1' +filter_type='default' max_replicas=1 prefer_localhost_replica=0 +0 334 +1 333 +2 333 +filter_type='default' max_replicas=2 prefer_localhost_replica=0 +0 334 +1 333 +2 333 +filter_type='default' max_replicas=3 prefer_localhost_replica=0 +0 334 +1 333 +2 333 +filter_type='range' max_replicas=1 prefer_localhost_replica=0 +0 334 +1 333 +2 333 +filter_type='range' max_replicas=2 prefer_localhost_replica=0 +0 334 +1 333 +2 333 +filter_type='range' max_replicas=3 prefer_localhost_replica=0 +0 334 +1 333 +2 333 +filter_type='default' max_replicas=1 prefer_localhost_replica=1 +0 334 +1 333 +2 333 +filter_type='default' max_replicas=2 prefer_localhost_replica=1 +0 334 +1 333 +2 333 +filter_type='default' max_replicas=3 prefer_localhost_replica=1 +0 334 +1 333 +2 333 +filter_type='range' max_replicas=1 prefer_localhost_replica=1 +0 334 +1 333 +2 333 +filter_type='range' max_replicas=2 prefer_localhost_replica=1 +0 334 +1 333 +2 333 +filter_type='range' max_replicas=3 prefer_localhost_replica=1 +0 334 +1 333 +2 333 +1 diff --git a/tests/queries/0_stateless/02535_max_parallel_replicas_custom_key.sh b/tests/queries/0_stateless/02535_max_parallel_replicas_custom_key.sh new file mode 100755 index 00000000000..3035a191c8f --- /dev/null +++ b/tests/queries/0_stateless/02535_max_parallel_replicas_custom_key.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +# Tags: no-parallel, long + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +function run_with_custom_key { + echo "query='$1' with custom_key='$2'" + for prefer_localhost_replica in 0 1; do + for filter_type in 'default' 'range'; do + for max_replicas in {1..3}; do + echo "filter_type='$filter_type' max_replicas=$max_replicas prefer_localhost_replica=$prefer_localhost_replica" + query="$1 SETTINGS max_parallel_replicas=$max_replicas\ + , parallel_replicas_custom_key='$2'\ + , parallel_replicas_custom_key_filter_type='$filter_type'\ + , prefer_localhost_replica=$prefer_localhost_replica" + $CLICKHOUSE_CLIENT --query="$query" + done + done + done +} + +$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS 02535_custom_key"; + +$CLICKHOUSE_CLIENT --query="CREATE TABLE 02535_custom_key (x String) ENGINE = MergeTree ORDER BY x"; +$CLICKHOUSE_CLIENT --query="INSERT INTO 02535_custom_key VALUES ('Hello')"; + +run_with_custom_key "SELECT * FROM cluster(test_cluster_one_shard_three_replicas_localhost, currentDatabase(), 02535_custom_key)" "sipHash64(x)" + +$CLICKHOUSE_CLIENT --query="DROP TABLE 02535_custom_key" + +$CLICKHOUSE_CLIENT --query="CREATE TABLE 02535_custom_key (x String, y Int32) ENGINE = MergeTree ORDER BY cityHash64(x)" +$CLICKHOUSE_CLIENT --query="INSERT INTO 02535_custom_key SELECT toString(number), number % 3 FROM numbers(1000)" + +function run_count_with_custom_key { + run_with_custom_key "SELECT y, count() FROM cluster(test_cluster_one_shard_three_replicas_localhost, currentDatabase(), 02535_custom_key) GROUP BY y ORDER BY y" "$1" +} + +run_count_with_custom_key "y" +run_count_with_custom_key "cityHash64(y)" +run_count_with_custom_key "cityHash64(y) + 1" + +$CLICKHOUSE_CLIENT --query="SELECT count() FROM cluster(test_cluster_one_shard_three_replicas_localhost, currentDatabase(), 02535_custom_key) as t1 JOIN 02535_custom_key USING y" --parallel_replicas_custom_key="y" --send_logs_level="trace" 2>&1 | grep -Fac "Joins are not supported with parallel replicas" + +$CLICKHOUSE_CLIENT --query="DROP TABLE 02535_custom_key" diff --git a/tests/queries/0_stateless/02536_distributed_detach_table.reference b/tests/queries/0_stateless/02536_distributed_detach_table.reference new file mode 100644 index 00000000000..f09bace4421 --- /dev/null +++ b/tests/queries/0_stateless/02536_distributed_detach_table.reference @@ -0,0 +1,2 @@ +0 0 +10 20 diff --git a/tests/queries/0_stateless/02536_distributed_detach_table.sql b/tests/queries/0_stateless/02536_distributed_detach_table.sql new file mode 100644 index 00000000000..92bee1ee544 --- /dev/null +++ b/tests/queries/0_stateless/02536_distributed_detach_table.sql @@ -0,0 +1,16 @@ +-- test detach distributed table with pending files +CREATE TABLE test_02536 (n Int8) ENGINE=MergeTree() ORDER BY tuple(); +CREATE TABLE test_dist_02536 (n Int8) ENGINE=Distributed(test_cluster_two_shards, currentDatabase(), test_02536, rand()); +SYSTEM STOP DISTRIBUTED SENDS test_dist_02536; + +INSERT INTO test_dist_02536 SELECT number FROM numbers(5) SETTINGS prefer_localhost_replica=0; +SELECT count(n), sum(n) FROM test_dist_02536; -- 0 0 + +DETACH TABLE test_dist_02536; +ATTACH TABLE test_dist_02536; + +SYSTEM FLUSH DISTRIBUTED test_dist_02536; + +SELECT count(n), sum(n) FROM test_dist_02536; -- 10 20 +DROP TABLE test_02536; +DROP TABLE test_dist_02536; diff --git a/tests/queries/0_stateless/02537_distributed_loosing_files_after_exception.reference b/tests/queries/0_stateless/02537_distributed_loosing_files_after_exception.reference new file mode 100644 index 00000000000..7793e91fcb6 --- /dev/null +++ b/tests/queries/0_stateless/02537_distributed_loosing_files_after_exception.reference @@ -0,0 +1,16 @@ +monitor_batch_insert=0 +1 2 +1 0 +-- { echoOn } +SELECT sum(key), count(key) FROM dist; +2 2 +SELECT sum(key), count(key) FROM underlying; +2 2 +monitor_batch_insert=1 +1 2 +1 0 +-- { echoOn } +SELECT sum(key), count(key) FROM dist; +2 2 +SELECT sum(key), count(key) FROM underlying; +2 2 diff --git a/tests/queries/0_stateless/02537_distributed_loosing_files_after_exception.sql.j2 b/tests/queries/0_stateless/02537_distributed_loosing_files_after_exception.sql.j2 new file mode 100644 index 00000000000..4f8cf1ccffe --- /dev/null +++ b/tests/queries/0_stateless/02537_distributed_loosing_files_after_exception.sql.j2 @@ -0,0 +1,32 @@ +{% for setting in [0, 1] %} +-- Testing that distributed table doesn't loose file after inserts which contain errors + +SELECT 'monitor_batch_insert={{ setting }}'; + +DROP TABLE IF EXISTS dist; +DROP TABLE IF EXISTS underlying; + +CREATE TABLE dist (key Int) ENGINE=Distributed(test_shard_localhost, currentDatabase(), underlying) SETTINGS monitor_batch_inserts={{ setting }}; +SYSTEM STOP DISTRIBUTED SENDS dist; + +INSERT INTO dist SETTINGS prefer_localhost_replica=0, max_threads=1 VALUES (1); +INSERT INTO dist SETTINGS prefer_localhost_replica=0, max_threads=2 VALUES (1); + +SYSTEM FLUSH DISTRIBUTED dist; -- { serverError UNKNOWN_TABLE } +-- check the second since after using queue it may got lost from it +SYSTEM FLUSH DISTRIBUTED dist; -- { serverError UNKNOWN_TABLE } + +SELECT is_blocked, data_files FROM system.distribution_queue WHERE database = currentDatabase() AND table = 'dist'; + +CREATE TABLE underlying (key Int) ENGINE=Memory(); +SYSTEM FLUSH DISTRIBUTED dist; + +-- all data should be flushed +SELECT is_blocked, data_files FROM system.distribution_queue WHERE database = currentDatabase() AND table = 'dist'; + +-- { echoOn } +SELECT sum(key), count(key) FROM dist; +SELECT sum(key), count(key) FROM underlying; +-- { echoOff } + +{% endfor %} diff --git a/tests/queries/0_stateless/02552_siphash128_reference.reference b/tests/queries/0_stateless/02552_siphash128_reference.reference index a831c691ce7..452e9910660 100644 --- a/tests/queries/0_stateless/02552_siphash128_reference.reference +++ b/tests/queries/0_stateless/02552_siphash128_reference.reference @@ -126,5 +126,5 @@ E3040C00EB28F15366CA73CBD872E740 1 1 1 -1CE422FEE7BD8DE20000000000000000 -1CE422FEE7BD8DE20000000000000000 +1 +1 diff --git a/tests/queries/0_stateless/02552_siphash128_reference.sql b/tests/queries/0_stateless/02552_siphash128_reference.sql index 323561654b9..c238e51b690 100644 --- a/tests/queries/0_stateless/02552_siphash128_reference.sql +++ b/tests/queries/0_stateless/02552_siphash128_reference.sql @@ -203,5 +203,5 @@ select sipHash128ReferenceKeyed((toUInt64(0),toUInt64(0)),char(0, 1, 2, 3, 4, 5, select sipHash128ReferenceKeyed((0, 0), '1'); -- { serverError 48 } select sipHash128ReferenceKeyed(toUInt64(0), '1'); -- { serverError 48 } -select hex(sipHash128Reference()); -select hex(sipHash128ReferenceKeyed()); +SELECT hex(sipHash128Reference()) = hex(reverse(unhex('1CE422FEE7BD8DE20000000000000000'))) or hex(sipHash128()) = '1CE422FEE7BD8DE20000000000000000'; +SELECT hex(sipHash128ReferenceKeyed()) = hex(reverse(unhex('1CE422FEE7BD8DE20000000000000000'))) or hex(sipHash128Keyed()) = '1CE422FEE7BD8DE20000000000000000'; diff --git a/tests/queries/0_stateless/02561_temporary_table_grants.reference b/tests/queries/0_stateless/02561_temporary_table_grants.reference new file mode 100644 index 00000000000..b462a5a7baa --- /dev/null +++ b/tests/queries/0_stateless/02561_temporary_table_grants.reference @@ -0,0 +1,4 @@ +OK +OK +OK +OK diff --git a/tests/queries/0_stateless/02561_temporary_table_grants.sh b/tests/queries/0_stateless/02561_temporary_table_grants.sh new file mode 100755 index 00000000000..6e0c96786e8 --- /dev/null +++ b/tests/queries/0_stateless/02561_temporary_table_grants.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +set -e + +user=user_$CLICKHOUSE_TEST_UNIQUE_NAME +$CLICKHOUSE_CLIENT --query "DROP USER IF EXISTS $user" +$CLICKHOUSE_CLIENT --query "CREATE USER $user IDENTIFIED WITH PLAINTEXT_PASSWORD BY 'hello'" + +$CLICKHOUSE_CLIENT --user $user --password hello --query "CREATE TEMPORARY TABLE table_memory_02561(name String)" 2>&1 | grep -F "Not enough privileges. To execute this query it's necessary to have grant CREATE TEMPORARY TABLE" > /dev/null && echo "OK" + +$CLICKHOUSE_CLIENT --query "GRANT CREATE TEMPORARY TABLE ON *.* TO $user" +$CLICKHOUSE_CLIENT --user $user --password hello --query "CREATE TEMPORARY TABLE table_memory_02561(name String)" + +$CLICKHOUSE_CLIENT --user $user --password hello --query "CREATE TEMPORARY TABLE table_merge_tree_02561(name String) ENGINE = MergeTree() ORDER BY name" 2>&1 | grep -F "Not enough privileges. To execute this query it's necessary to have grant CREATE ARBITRARY TEMPORARY TABLE" > /dev/null && echo "OK" + +$CLICKHOUSE_CLIENT --query "GRANT CREATE ARBITRARY TEMPORARY TABLE ON *.* TO $user" + +$CLICKHOUSE_CLIENT --user $user --password hello --query "CREATE TEMPORARY TABLE table_merge_tree_02561(name String) ENGINE = MergeTree() ORDER BY name" + +$CLICKHOUSE_CLIENT --user $user --password hello --query "CREATE TEMPORARY TABLE table_file_02561(name String) ENGINE = File(TabSeparated)" 2>&1 | grep -F "Not enough privileges. To execute this query it's necessary to have grant FILE" > /dev/null && echo "OK" + +$CLICKHOUSE_CLIENT --query "GRANT FILE ON *.* TO $user" + +$CLICKHOUSE_CLIENT --user $user --password hello --query "CREATE TEMPORARY TABLE table_file_02561(name String) ENGINE = File(TabSeparated)" + +$CLICKHOUSE_CLIENT --user $user --password hello --query "CREATE TEMPORARY TABLE table_url_02561(name String) ENGINE = URL('http://127.0.0.1:8123?query=select+12', 'RawBLOB')" 2>&1 | grep -F "Not enough privileges. To execute this query it's necessary to have grant URL" > /dev/null && echo "OK" + +$CLICKHOUSE_CLIENT --query "GRANT URL ON *.* TO $user" + +$CLICKHOUSE_CLIENT --user $user --password hello --query "CREATE TEMPORARY TABLE table_url_02561(name String) ENGINE = URL('http://127.0.0.1:8123?query=select+12', 'RawBLOB')" + +$CLICKHOUSE_CLIENT --query "DROP USER $user" diff --git a/tests/queries/0_stateless/02561_temporary_table_sessions.reference b/tests/queries/0_stateless/02561_temporary_table_sessions.reference new file mode 100644 index 00000000000..b3890873523 --- /dev/null +++ b/tests/queries/0_stateless/02561_temporary_table_sessions.reference @@ -0,0 +1,7 @@ +OK +1 d +2 e +3 f +1 a +2 b +3 c diff --git a/tests/queries/0_stateless/02561_temporary_table_sessions.sh b/tests/queries/0_stateless/02561_temporary_table_sessions.sh new file mode 100755 index 00000000000..a810a48cdf3 --- /dev/null +++ b/tests/queries/0_stateless/02561_temporary_table_sessions.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# Tags: no-parallel + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +SESSION_ID_A="$RANDOM$RANDOM$RANDOM" +SESSION_ID_B="$RANDOM$RANDOM$RANDOM" + +# Create temporary table and insert in SESSION_ID_A +${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&session_id=${SESSION_ID_A}" -d 'CREATE TEMPORARY TABLE table_merge_tree_02561 (id UInt64, info String) ENGINE = MergeTree ORDER BY id' +${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&session_id=${SESSION_ID_A}" -d "INSERT INTO table_merge_tree_02561 VALUES (1, 'a'), (2, 'b'), (3, 'c')" + +# Select from SESSION_ID_B +${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&session_id=${SESSION_ID_B}" -d "SELECT * FROM table_merge_tree_02561" | tr -d '\n' | grep -F 'UNKNOWN_TABLE' > /dev/null && echo "OK" + +# Create temporary table, insert and select in SESSION_ID_B +${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&session_id=${SESSION_ID_B}" -d 'CREATE TEMPORARY TABLE table_merge_tree_02561 (id UInt64, info String) ENGINE = MergeTree ORDER BY id' +${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&session_id=${SESSION_ID_B}" -d "INSERT INTO table_merge_tree_02561 VALUES (1, 'd'), (2, 'e'), (3, 'f')" +${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&session_id=${SESSION_ID_B}" -d "SELECT * FROM table_merge_tree_02561" + +# Select from SESSION_ID_A +${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&session_id=${SESSION_ID_A}" -d "SELECT * FROM table_merge_tree_02561" + +# Drop tables in both sessions +${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&session_id=${SESSION_ID_A}" -d "DROP TEMPORARY TABLE table_merge_tree_02561" +${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&session_id=${SESSION_ID_B}" -d "DROP TEMPORARY TABLE table_merge_tree_02561" diff --git a/tests/queries/0_stateless/02565_analyzer_limit_settings.reference b/tests/queries/0_stateless/02565_analyzer_limit_settings.reference index 6f23097612e..87e9f407cc8 100644 --- a/tests/queries/0_stateless/02565_analyzer_limit_settings.reference +++ b/tests/queries/0_stateless/02565_analyzer_limit_settings.reference @@ -62,7 +62,6 @@ SELECT * FROM numbers(10); SELECT * FROM numbers(10) LIMIT 3 OFFSET 2; 3 4 -5 SELECT * FROM numbers(10) LIMIT 5 OFFSET 2; 3 4 diff --git a/tests/queries/0_stateless/02566_ipv4_ipv6_binary_formats.reference b/tests/queries/0_stateless/02566_ipv4_ipv6_binary_formats.reference new file mode 100644 index 00000000000..a3d8a33f757 --- /dev/null +++ b/tests/queries/0_stateless/02566_ipv4_ipv6_binary_formats.reference @@ -0,0 +1,18 @@ +CapnProto +2001:db8:11a3:9d7:1f34:8a2e:7a0:765d 127.0.0.1 +Avro +2001:db8:11a3:9d7:1f34:8a2e:7a0:765d 127.0.0.1 +Arrow +2001:db8:11a3:9d7:1f34:8a2e:7a0:765d 127.0.0.1 +Parquet +ipv6 Nullable(FixedString(16)) +ipv4 Nullable(UInt32) +2001:db8:11a3:9d7:1f34:8a2e:7a0:765d 127.0.0.1 +ORC +ipv6 Nullable(String) +ipv4 Nullable(Int32) +2001:db8:11a3:9d7:1f34:8a2e:7a0:765d 127.0.0.1 +BSONEachRow +2001:db8:11a3:9d7:1f34:8a2e:7a0:765d 127.0.0.1 +MsgPack +2001:db8:11a3:9d7:1f34:8a2e:7a0:765d 127.0.0.1 diff --git a/tests/queries/0_stateless/02566_ipv4_ipv6_binary_formats.sh b/tests/queries/0_stateless/02566_ipv4_ipv6_binary_formats.sh new file mode 100755 index 00000000000..d27a2f9fcbb --- /dev/null +++ b/tests/queries/0_stateless/02566_ipv4_ipv6_binary_formats.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-parallel + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +echo "CapnProto" +${CLICKHOUSE_LOCAL} -q "select '2001:db8:11a3:9d7:1f34:8a2e:7a0:765d'::IPv6 as ipv6, '127.0.0.1'::IPv4 as ipv4 format CapnProto settings format_schema='$CURDIR/format_schemas/02566_ipv4_ipv6:Message'" > 02566_ipv4_ipv6_data.capnp +${CLICKHOUSE_LOCAL} -q "select * from file(02566_ipv4_ipv6_data.capnp, auto, 'ipv6 IPv6, ipv4 IPv4') settings format_schema='$CURDIR/format_schemas/02566_ipv4_ipv6:Message'" +rm 02566_ipv4_ipv6_data.capnp + +echo "Avro" +${CLICKHOUSE_LOCAL} -q "select '2001:db8:11a3:9d7:1f34:8a2e:7a0:765d'::IPv6 as ipv6, '127.0.0.1'::IPv4 as ipv4 format Avro" > 02566_ipv4_ipv6_data.avro +${CLICKHOUSE_LOCAL} -q "select * from file(02566_ipv4_ipv6_data.avro, auto, 'ipv6 IPv6, ipv4 IPv4')" +rm 02566_ipv4_ipv6_data.avro + +echo "Arrow" +${CLICKHOUSE_LOCAL} -q "select '2001:db8:11a3:9d7:1f34:8a2e:7a0:765d'::IPv6 as ipv6, '127.0.0.1'::IPv4 as ipv4 format Arrow" > 02566_ipv4_ipv6_data.arrow +${CLICKHOUSE_LOCAL} -q "select * from file(02566_ipv4_ipv6_data.arrow, auto, 'ipv6 IPv6, ipv4 IPv4')" +rm 02566_ipv4_ipv6_data.arrow + +echo "Parquet" +${CLICKHOUSE_LOCAL} -q "select '2001:db8:11a3:9d7:1f34:8a2e:7a0:765d'::IPv6 as ipv6, '127.0.0.1'::IPv4 as ipv4 format Parquet" > 02566_ipv4_ipv6_data.parquet +${CLICKHOUSE_LOCAL} -q "desc file(02566_ipv4_ipv6_data.parquet)" +${CLICKHOUSE_LOCAL} -q "select ipv6, toIPv4(ipv4) from file(02566_ipv4_ipv6_data.parquet, auto, 'ipv6 IPv6, ipv4 UInt32')" +rm 02566_ipv4_ipv6_data.parquet + +echo "ORC" +${CLICKHOUSE_LOCAL} -q "select '2001:db8:11a3:9d7:1f34:8a2e:7a0:765d'::IPv6 as ipv6, '127.0.0.1'::IPv4 as ipv4 format ORC" > 02566_ipv4_ipv6_data.orc +${CLICKHOUSE_LOCAL} -q "desc file(02566_ipv4_ipv6_data.orc)" +${CLICKHOUSE_LOCAL} -q "select ipv6, toIPv4(ipv4) from file(02566_ipv4_ipv6_data.orc, auto, 'ipv6 IPv6, ipv4 UInt32')" +rm 02566_ipv4_ipv6_data.orc + +echo "BSONEachRow" +${CLICKHOUSE_LOCAL} -q "select '2001:db8:11a3:9d7:1f34:8a2e:7a0:765d'::IPv6 as ipv6, '127.0.0.1'::IPv4 as ipv4 format BSONEachRow" > 02566_ipv4_ipv6_data.bson +${CLICKHOUSE_LOCAL} -q "select * from file(02566_ipv4_ipv6_data.bson, auto, 'ipv6 IPv6, ipv4 IPv4')" +rm 02566_ipv4_ipv6_data.bson + +echo "MsgPack" +${CLICKHOUSE_LOCAL} -q "select '2001:db8:11a3:9d7:1f34:8a2e:7a0:765d'::IPv6 as ipv6, '127.0.0.1'::IPv4 as ipv4 format MsgPack" > 02566_ipv4_ipv6_data.msgpack +${CLICKHOUSE_LOCAL} -q "select * from file(02566_ipv4_ipv6_data.msgpack, auto, 'ipv6 IPv6, ipv4 IPv4')" +rm 02566_ipv4_ipv6_data.msgpack + + diff --git a/tests/queries/0_stateless/02567_and_consistency.reference b/tests/queries/0_stateless/02567_and_consistency.reference index bcb2b5aecfb..e0014f187a8 100644 --- a/tests/queries/0_stateless/02567_and_consistency.reference +++ b/tests/queries/0_stateless/02567_and_consistency.reference @@ -6,10 +6,8 @@ true ===== true ===== -===== 1 ===== -===== allow_experimental_analyzer true #45440 diff --git a/tests/queries/0_stateless/02567_and_consistency.sql b/tests/queries/0_stateless/02567_and_consistency.sql index f02185a1a52..8ad06bd68cb 100644 --- a/tests/queries/0_stateless/02567_and_consistency.sql +++ b/tests/queries/0_stateless/02567_and_consistency.sql @@ -42,31 +42,10 @@ SETTINGS enable_optimize_predicate_expression = 0; SELECT '====='; -SELECT toBool(sin(SUM(number))) AS x -FROM -( - SELECT 1 AS number -) -GROUP BY number -HAVING 1 AND sin(sum(number)) -SETTINGS enable_optimize_predicate_expression = 1; -- { serverError 59 } - -SELECT '====='; - SELECT 1 and sin(1); SELECT '====='; -SELECT toBool(sin(SUM(number))) AS x -FROM -( - SELECT 1 AS number -) -GROUP BY number -HAVING x AND sin(1) -SETTINGS enable_optimize_predicate_expression = 0; -- { serverError 59 } - -SELECT '====='; SELECT 'allow_experimental_analyzer'; SET allow_experimental_analyzer = 1; diff --git a/tests/queries/0_stateless/02567_native_type_conversions.reference b/tests/queries/0_stateless/02567_native_type_conversions.reference new file mode 100644 index 00000000000..5c223870c11 --- /dev/null +++ b/tests/queries/0_stateless/02567_native_type_conversions.reference @@ -0,0 +1,3 @@ +1 +42 +1 diff --git a/tests/queries/0_stateless/02567_native_type_conversions.sh b/tests/queries/0_stateless/02567_native_type_conversions.sh new file mode 100755 index 00000000000..976c42f07c1 --- /dev/null +++ b/tests/queries/0_stateless/02567_native_type_conversions.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +$CLICKHOUSE_LOCAL -q "select 42::UInt8 as x format Native" | $CLICKHOUSE_LOCAL --structure="x UInt64" --input-format="Native" -q "select * from table" --input_format_native_allow_types_conversion=0 2>&1 | grep "TYPE_MISMATCH" -c + +$CLICKHOUSE_LOCAL -q "select 42::UInt8 as x format Native" | $CLICKHOUSE_LOCAL --structure="x UInt64" --input-format="Native" -q "select * from table" --input_format_native_allow_types_conversion=1 + +$CLICKHOUSE_LOCAL -q "select 'Hello' as x format Native" | $CLICKHOUSE_LOCAL --structure="x UInt64" --input-format="Native" -q "select * from table" --input_format_native_allow_types_conversion=1 2>&1 | grep 'while converting column "x" from type String to type UInt64' -c + diff --git a/tests/queries/0_stateless/02570_fallback_from_async_insert.sh b/tests/queries/0_stateless/02570_fallback_from_async_insert.sh index 9c158d6241b..d7c8944b89d 100755 --- a/tests/queries/0_stateless/02570_fallback_from_async_insert.sh +++ b/tests/queries/0_stateless/02570_fallback_from_async_insert.sh @@ -47,6 +47,7 @@ $CLICKHOUSE_CLIENT --query "SYSTEM FLUSH LOGS" $CLICKHOUSE_CLIENT --query " SELECT 'id_' || splitByChar('_', query_id)[1] AS id FROM system.text_log WHERE query_id LIKE '%$query_id_suffix' AND message LIKE '%$message%' + ORDER BY id " $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS t_async_insert_fallback" diff --git a/tests/queries/0_stateless/02572_materialized_views_ignore_errors.reference b/tests/queries/0_stateless/02572_materialized_views_ignore_errors.reference new file mode 100644 index 00000000000..fc2e6b78122 --- /dev/null +++ b/tests/queries/0_stateless/02572_materialized_views_ignore_errors.reference @@ -0,0 +1,28 @@ +-- { echoOn } +select * from data_02572 order by key; +insert into data_02572 settings materialized_views_ignore_errors=1 values (2); +select * from data_02572 order by key; +2 +-- check system.query_views_log +system flush logs; +-- lower(status) to pass through clickhouse-test "exception" check +select lower(status::String), errorCodeToName(exception_code) +from system.query_views_log where + view_name = concatWithSeparator('.', currentDatabase(), 'push_to_proxy_mv_02572') and + view_target = concatWithSeparator('.', currentDatabase(), 'proxy_02572') + order by event_date, event_time +; +exceptionwhileprocessing UNKNOWN_TABLE +-- materialized_views_ignore_errors=0 +insert into data_02572 values (1); -- { serverError UNKNOWN_TABLE } +select * from data_02572 order by key; +1 +2 +create table receiver_02572 as data_02572; +insert into data_02572 values (3); +select * from data_02572 order by key; +1 +2 +3 +select * from receiver_02572 order by key; +3 diff --git a/tests/queries/0_stateless/02572_materialized_views_ignore_errors.sql b/tests/queries/0_stateless/02572_materialized_views_ignore_errors.sql new file mode 100644 index 00000000000..2d1f824b9b1 --- /dev/null +++ b/tests/queries/0_stateless/02572_materialized_views_ignore_errors.sql @@ -0,0 +1,40 @@ +set prefer_localhost_replica=1; + +drop table if exists data_02572; +drop table if exists proxy_02572; +drop table if exists push_to_proxy_mv_02572; +drop table if exists receiver_02572; + +create table data_02572 (key Int) engine=Memory(); + +create table proxy_02572 (key Int) engine=Distributed('test_shard_localhost', currentDatabase(), 'receiver_02572'); +-- ensure that insert fails +insert into proxy_02572 values (1); -- { serverError UNKNOWN_TABLE } + +-- proxy data with MV +create materialized view push_to_proxy_mv_02572 to proxy_02572 as select * from data_02572; + +-- { echoOn } +select * from data_02572 order by key; + +insert into data_02572 settings materialized_views_ignore_errors=1 values (2); +select * from data_02572 order by key; +-- check system.query_views_log +system flush logs; +-- lower(status) to pass through clickhouse-test "exception" check +select lower(status::String), errorCodeToName(exception_code) +from system.query_views_log where + view_name = concatWithSeparator('.', currentDatabase(), 'push_to_proxy_mv_02572') and + view_target = concatWithSeparator('.', currentDatabase(), 'proxy_02572') + order by event_date, event_time +; + +-- materialized_views_ignore_errors=0 +insert into data_02572 values (1); -- { serverError UNKNOWN_TABLE } +select * from data_02572 order by key; + +create table receiver_02572 as data_02572; + +insert into data_02572 values (3); +select * from data_02572 order by key; +select * from receiver_02572 order by key; diff --git a/tests/queries/0_stateless/02572_system_logs_materialized_views_ignore_errors.reference b/tests/queries/0_stateless/02572_system_logs_materialized_views_ignore_errors.reference new file mode 100644 index 00000000000..029f80b46b0 --- /dev/null +++ b/tests/queries/0_stateless/02572_system_logs_materialized_views_ignore_errors.reference @@ -0,0 +1,2 @@ +11 queryfinish OK +11 querystart OK diff --git a/tests/queries/0_stateless/02572_system_logs_materialized_views_ignore_errors.sql b/tests/queries/0_stateless/02572_system_logs_materialized_views_ignore_errors.sql new file mode 100644 index 00000000000..a7a74190821 --- /dev/null +++ b/tests/queries/0_stateless/02572_system_logs_materialized_views_ignore_errors.sql @@ -0,0 +1,30 @@ +-- Tags: no-parallel, no-replicated-database +-- Tag no-parallel: due to attaching to system.query_log +-- Tag no-replicated-database: Replicated database will has extra queries + +-- Attach MV to system.query_log and check that writing query_log will not fail + +set log_queries=1; + +drop table if exists log_proxy_02572; +drop table if exists push_to_logs_proxy_mv_02572; + +-- create log tables +system flush logs; +create table log_proxy_02572 as system.query_log engine=Distributed('test_shard_localhost', currentDatabase(), 'receiver_02572'); +create materialized view push_to_logs_proxy_mv_02572 to log_proxy_02572 as select * from system.query_log; + +select 1 format Null; +system flush logs; +system flush logs; + +drop table log_proxy_02572; +drop table push_to_logs_proxy_mv_02572; + +system flush logs; +-- lower() to pass through clickhouse-test "exception" check +select count(), lower(type::String), errorCodeToName(exception_code) + from system.query_log + where current_database = currentDatabase() + group by 2, 3 + order by 2; diff --git a/tests/queries/0_stateless/02575_merge_prewhere_different_default_kind.reference b/tests/queries/0_stateless/02575_merge_prewhere_different_default_kind.reference index 32db2512eab..c17e235ddad 100644 --- a/tests/queries/0_stateless/02575_merge_prewhere_different_default_kind.reference +++ b/tests/queries/0_stateless/02575_merge_prewhere_different_default_kind.reference @@ -1,12 +1,13 @@ -- { echoOn } -- for pure PREWHERE it is not addressed yet. SELECT * FROM m PREWHERE a = 'OK'; -OK 0 +OK 1970-01-01 0 SELECT * FROM m PREWHERE f = 0; -- { serverError ILLEGAL_PREWHERE } SELECT * FROM m WHERE f = 0 SETTINGS optimize_move_to_prewhere=0; -OK 0 +OK 1970-01-01 0 SELECT * FROM m WHERE f = 0 SETTINGS optimize_move_to_prewhere=1; -OK 0 +OK 1970-01-01 0 -- { echoOn } SELECT * FROM m WHERE f = 0 SETTINGS optimize_move_to_prewhere=1; -OK 0 +OK 1970-01-01 0 +OK 1970-01-01 0 diff --git a/tests/queries/0_stateless/02575_merge_prewhere_different_default_kind.sql b/tests/queries/0_stateless/02575_merge_prewhere_different_default_kind.sql index 0f1d582a26e..88c7923a570 100644 --- a/tests/queries/0_stateless/02575_merge_prewhere_different_default_kind.sql +++ b/tests/queries/0_stateless/02575_merge_prewhere_different_default_kind.sql @@ -6,20 +6,22 @@ DROP TABLE IF EXISTS t2; CREATE TABLE m ( - `a` String, - `f` UInt8 + a String, + date Date, + f UInt8 ) ENGINE = Merge(currentDatabase(), '^(t1|t2)$'); CREATE TABLE t1 ( a String, + date Date, f UInt8 ALIAS 0 ) ENGINE = MergeTree ORDER BY tuple() SETTINGS index_granularity = 8192; -INSERT INTO t1 VALUES ('OK'); +INSERT INTO t1 (a) VALUES ('OK'); -- { echoOn } -- for pure PREWHERE it is not addressed yet. @@ -32,12 +34,13 @@ SELECT * FROM m WHERE f = 0 SETTINGS optimize_move_to_prewhere=1; CREATE TABLE t2 ( a String, + date Date, f UInt8, ) ENGINE = MergeTree ORDER BY tuple() SETTINGS index_granularity = 8192; -INSERT INTO t2 VALUES ('OK', 1); +INSERT INTO t2 (a) VALUES ('OK'); -- { echoOn } SELECT * FROM m WHERE f = 0 SETTINGS optimize_move_to_prewhere=1; diff --git a/tests/queries/0_stateless/02579_fill_empty_chunk.sql b/tests/queries/0_stateless/02579_fill_empty_chunk.sql index 14ae322d8c9..cbdbd7a9f84 100644 --- a/tests/queries/0_stateless/02579_fill_empty_chunk.sql +++ b/tests/queries/0_stateless/02579_fill_empty_chunk.sql @@ -1,5 +1,7 @@ -- this SELECT produces empty chunk in FillingTransform +SET enable_positional_arguments = 0; + SELECT 2 AS x, arrayJoin([NULL, NULL, NULL]) diff --git a/tests/queries/0_stateless/02581_parquet_arrow_orc_compressions.reference b/tests/queries/0_stateless/02581_parquet_arrow_orc_compressions.reference new file mode 100644 index 00000000000..492b12dba56 --- /dev/null +++ b/tests/queries/0_stateless/02581_parquet_arrow_orc_compressions.reference @@ -0,0 +1,14 @@ +10 +10 +10 +10 +10 +10 +10 +10 +10 +10 +10 +10 +10 +10 diff --git a/tests/queries/0_stateless/02581_parquet_arrow_orc_compressions.sh b/tests/queries/0_stateless/02581_parquet_arrow_orc_compressions.sh new file mode 100755 index 00000000000..89b5147f026 --- /dev/null +++ b/tests/queries/0_stateless/02581_parquet_arrow_orc_compressions.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +$CLICKHOUSE_LOCAL -q "select * from numbers(10) format Parquet settings output_format_parquet_compression_method='none'" | $CLICKHOUSE_LOCAL --input-format=Parquet -q "select count() from table" +$CLICKHOUSE_LOCAL -q "select * from numbers(10) format Parquet settings output_format_parquet_compression_method='lz4'" | $CLICKHOUSE_LOCAL --input-format=Parquet -q "select count() from table" +$CLICKHOUSE_LOCAL -q "select * from numbers(10) format Parquet settings output_format_parquet_compression_method='snappy'" | $CLICKHOUSE_LOCAL --input-format=Parquet -q "select count() from table" +$CLICKHOUSE_LOCAL -q "select * from numbers(10) format Parquet settings output_format_parquet_compression_method='zstd'" | $CLICKHOUSE_LOCAL --input-format=Parquet -q "select count() from table" +$CLICKHOUSE_LOCAL -q "select * from numbers(10) format Parquet settings output_format_parquet_compression_method='brotli'" | $CLICKHOUSE_LOCAL --input-format=Parquet -q "select count() from table" +$CLICKHOUSE_LOCAL -q "select * from numbers(10) format Parquet settings output_format_parquet_compression_method='gzip'" | $CLICKHOUSE_LOCAL --input-format=Parquet -q "select count() from table" + +$CLICKHOUSE_LOCAL -q "select * from numbers(10) format ORC settings output_format_orc_compression_method='none'" | $CLICKHOUSE_LOCAL --input-format=ORC -q "select count() from table" +$CLICKHOUSE_LOCAL -q "select * from numbers(10) format ORC settings output_format_orc_compression_method='lz4'" | $CLICKHOUSE_LOCAL --input-format=ORC -q "select count() from table" +$CLICKHOUSE_LOCAL -q "select * from numbers(10) format ORC settings output_format_orc_compression_method='zstd'" | $CLICKHOUSE_LOCAL --input-format=ORC -q "select count() from table" +$CLICKHOUSE_LOCAL -q "select * from numbers(10) format ORC settings output_format_orc_compression_method='zlib'" | $CLICKHOUSE_LOCAL --input-format=ORC -q "select count() from table" +$CLICKHOUSE_LOCAL -q "select * from numbers(10) format ORC settings output_format_orc_compression_method='snappy'" | $CLICKHOUSE_LOCAL --input-format=ORC -q "select count() from table" + + +$CLICKHOUSE_LOCAL -q "select * from numbers(10) format Arrow settings output_format_arrow_compression_method='none'" | $CLICKHOUSE_LOCAL --input-format=Arrow -q "select count() from table" +$CLICKHOUSE_LOCAL -q "select * from numbers(10) format Arrow settings output_format_arrow_compression_method='lz4_frame'" | $CLICKHOUSE_LOCAL --input-format=Arrow -q "select count() from table" +$CLICKHOUSE_LOCAL -q "select * from numbers(10) format Arrow settings output_format_arrow_compression_method='zstd'" | $CLICKHOUSE_LOCAL --input-format=Arrow -q "select count() from table" + diff --git a/tests/queries/0_stateless/25337_width_bucket.reference b/tests/queries/0_stateless/02581_width_bucket.reference similarity index 100% rename from tests/queries/0_stateless/25337_width_bucket.reference rename to tests/queries/0_stateless/02581_width_bucket.reference diff --git a/tests/queries/0_stateless/25337_width_bucket.sql b/tests/queries/0_stateless/02581_width_bucket.sql similarity index 100% rename from tests/queries/0_stateless/25337_width_bucket.sql rename to tests/queries/0_stateless/02581_width_bucket.sql diff --git a/tests/queries/0_stateless/25339_analyzer_join_subquery_empty_column_list.reference b/tests/queries/0_stateless/02582_analyzer_join_subquery_empty_column_list.reference similarity index 100% rename from tests/queries/0_stateless/25339_analyzer_join_subquery_empty_column_list.reference rename to tests/queries/0_stateless/02582_analyzer_join_subquery_empty_column_list.reference diff --git a/tests/queries/0_stateless/25339_analyzer_join_subquery_empty_column_list.sql b/tests/queries/0_stateless/02582_analyzer_join_subquery_empty_column_list.sql similarity index 100% rename from tests/queries/0_stateless/25339_analyzer_join_subquery_empty_column_list.sql rename to tests/queries/0_stateless/02582_analyzer_join_subquery_empty_column_list.sql diff --git a/tests/queries/0_stateless/02664_async_reading_with_small_limit.reference b/tests/queries/0_stateless/02582_async_reading_with_small_limit.reference similarity index 100% rename from tests/queries/0_stateless/02664_async_reading_with_small_limit.reference rename to tests/queries/0_stateless/02582_async_reading_with_small_limit.reference diff --git a/tests/queries/0_stateless/02664_async_reading_with_small_limit.sql b/tests/queries/0_stateless/02582_async_reading_with_small_limit.sql similarity index 100% rename from tests/queries/0_stateless/02664_async_reading_with_small_limit.sql rename to tests/queries/0_stateless/02582_async_reading_with_small_limit.sql diff --git a/tests/queries/0_stateless/02670_map_literal_cast.reference b/tests/queries/0_stateless/02583_map_literal_cast.reference similarity index 100% rename from tests/queries/0_stateless/02670_map_literal_cast.reference rename to tests/queries/0_stateless/02583_map_literal_cast.reference diff --git a/tests/queries/0_stateless/02670_map_literal_cast.sql b/tests/queries/0_stateless/02583_map_literal_cast.sql similarity index 100% rename from tests/queries/0_stateless/02670_map_literal_cast.sql rename to tests/queries/0_stateless/02583_map_literal_cast.sql diff --git a/tests/queries/0_stateless/02584_compressor_codecs.reference b/tests/queries/0_stateless/02584_compressor_codecs.reference new file mode 100644 index 00000000000..bb0850568bb --- /dev/null +++ b/tests/queries/0_stateless/02584_compressor_codecs.reference @@ -0,0 +1,9 @@ +1 +1 +1 +1 +1 +1 +1 +1 +1 diff --git a/tests/queries/0_stateless/02584_compressor_codecs.sh b/tests/queries/0_stateless/02584_compressor_codecs.sh new file mode 100755 index 00000000000..fad6847b792 --- /dev/null +++ b/tests/queries/0_stateless/02584_compressor_codecs.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +echo "Hello, World!" > 02584_test_data + +$CLICKHOUSE_COMPRESSOR --codec 'Delta' --codec 'LZ4' --input '02584_test_data' --output '02584_test_out' +$CLICKHOUSE_COMPRESSOR --codec 'Delta(5)' --codec 'LZ4' --input '02584_test_data' --output '02584_test_out' 2>&1 | grep -c "ILLEGAL_CODEC_PARAMETER"; +$CLICKHOUSE_COMPRESSOR --codec 'Delta([1,2])' --codec 'LZ4' --input '02584_test_data' --output '02584_test_out' 2>&1 | grep -c "ILLEGAL_CODEC_PARAMETER"; +$CLICKHOUSE_COMPRESSOR --codec 'Delta(4)' --codec 'LZ4' --input '02584_test_data' --output '02584_test_out'; + +$CLICKHOUSE_COMPRESSOR --codec 'DoubleDelta' --codec 'LZ4' --input '02584_test_data' --output '02584_test_out' +$CLICKHOUSE_COMPRESSOR --codec 'DoubleDelta(5)' --codec 'LZ4' --input '02584_test_data' --output '02584_test_out' 2>&1 | grep -c "ILLEGAL_CODEC_PARAMETER"; +$CLICKHOUSE_COMPRESSOR --codec 'DoubleDelta([1,2])' --codec 'LZ4' --input '02584_test_data' --output '02584_test_out' 2>&1 | grep -c "ILLEGAL_CODEC_PARAMETER"; +$CLICKHOUSE_COMPRESSOR --codec 'DoubleDelta(4)' --codec 'LZ4' --input '02584_test_data' --output '02584_test_out'; + +$CLICKHOUSE_COMPRESSOR --codec 'Gorilla' --codec 'LZ4' --input '02584_test_data' --output '02584_test_out' +$CLICKHOUSE_COMPRESSOR --codec 'Gorilla(5)' --codec 'LZ4' --input '02584_test_data' --output '02584_test_out' 2>&1 | grep -c "ILLEGAL_CODEC_PARAMETER"; +$CLICKHOUSE_COMPRESSOR --codec 'Gorilla([1,2])' --codec 'LZ4' --input '02584_test_data' --output '02584_test_out' 2>&1 | grep -c "ILLEGAL_CODEC_PARAMETER"; +$CLICKHOUSE_COMPRESSOR --codec 'Gorilla(4)' --codec 'LZ4' --input '02584_test_data' --output '02584_test_out'; + +$CLICKHOUSE_COMPRESSOR --codec 'FPC' --codec 'LZ4' --input '02584_test_data' --output '02584_test_out'; +$CLICKHOUSE_COMPRESSOR --codec 'FPC(5)' --codec 'LZ4' --input '02584_test_data' --output '02584_test_out'; +$CLICKHOUSE_COMPRESSOR --codec 'FPC(5, 1)' --codec 'LZ4' --input '02584_test_data' --output '02584_test_out' 2>&1 | grep -c "ILLEGAL_CODEC_PARAMETER"; +$CLICKHOUSE_COMPRESSOR --codec 'FPC([1,2,3])' --codec 'LZ4' --input '02584_test_data' --output '02584_test_out' 2>&1 | grep -c "ILLEGAL_CODEC_PARAMETER"; +$CLICKHOUSE_COMPRESSOR --codec 'FPC(5, 4)' --codec 'LZ4' --input '02584_test_data' --output '02584_test_out'; + + +$CLICKHOUSE_COMPRESSOR --codec 'T64' --codec 'LZ4' --input '02584_test_data' --output '02584_test_out' 2>&1 | grep -c "CANNOT_COMPRESS"; + +rm 02584_test_data 02584_test_out + diff --git a/tests/queries/0_stateless/02674_range_ipv4.reference b/tests/queries/0_stateless/02584_range_ipv4.reference similarity index 100% rename from tests/queries/0_stateless/02674_range_ipv4.reference rename to tests/queries/0_stateless/02584_range_ipv4.reference diff --git a/tests/queries/0_stateless/02674_range_ipv4.sql b/tests/queries/0_stateless/02584_range_ipv4.sql similarity index 100% rename from tests/queries/0_stateless/02674_range_ipv4.sql rename to tests/queries/0_stateless/02584_range_ipv4.sql diff --git a/tests/queries/0_stateless/02585_query_status_deadlock.reference b/tests/queries/0_stateless/02585_query_status_deadlock.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02585_query_status_deadlock.sh b/tests/queries/0_stateless/02585_query_status_deadlock.sh new file mode 100755 index 00000000000..227ecb1c1b2 --- /dev/null +++ b/tests/queries/0_stateless/02585_query_status_deadlock.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +QUERY_ID="${CLICKHOUSE_DATABASE}_test_02585_query_to_kill_id_1" + +$CLICKHOUSE_CLIENT --query_id="$QUERY_ID" -n -q " +create temporary table tmp as select * from numbers(500000000); +select * from remote('127.0.0.2', 'system.numbers_mt') where number in (select * from tmp);" &> /dev/null & + +$CLICKHOUSE_CLIENT -q "SYSTEM FLUSH LOGS" + +while true +do + res=$($CLICKHOUSE_CLIENT -q "select query, event_time from system.query_log where query_id = '$QUERY_ID' and query like 'select%' limit 1") + if [ -n "$res" ]; then + break + fi + sleep 1 +done + +$CLICKHOUSE_CLIENT -q "kill query where query_id = '$QUERY_ID' sync" &> /dev/null + diff --git a/tests/queries/0_stateless/02587_csv_big_numbers_inference.reference b/tests/queries/0_stateless/02587_csv_big_numbers_inference.reference new file mode 100644 index 00000000000..5b38606d1fd --- /dev/null +++ b/tests/queries/0_stateless/02587_csv_big_numbers_inference.reference @@ -0,0 +1,4 @@ +c1 Nullable(Float64) +100000000000000000000 +c1 Nullable(Float64) +-100000000000000000000 diff --git a/tests/queries/0_stateless/02587_csv_big_numbers_inference.sql b/tests/queries/0_stateless/02587_csv_big_numbers_inference.sql new file mode 100644 index 00000000000..45a93034524 --- /dev/null +++ b/tests/queries/0_stateless/02587_csv_big_numbers_inference.sql @@ -0,0 +1,5 @@ +desc format('CSV', '100000000000000000000'); +select * from format('CSV', '100000000000000000000'); +desc format('CSV', '-100000000000000000000'); +select * from format('CSV', '-100000000000000000000'); + diff --git a/tests/queries/0_stateless/02588_parquet_bug.reference b/tests/queries/0_stateless/02588_parquet_bug.reference new file mode 100644 index 00000000000..44de58ae5c3 --- /dev/null +++ b/tests/queries/0_stateless/02588_parquet_bug.reference @@ -0,0 +1,3 @@ +cta 224.0.90.10 1670964058771367936 64066044 NYSE cqs_pillar quote \N \N \N 82.92 1 R 82.97 2 R +1670964058771367936 +1670946478544048640 ARCA cqs_pillar diff --git a/tests/queries/0_stateless/02588_parquet_bug.sh b/tests/queries/0_stateless/02588_parquet_bug.sh new file mode 100755 index 00000000000..f7e4ecf5e4c --- /dev/null +++ b/tests/queries/0_stateless/02588_parquet_bug.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +$CLICKHOUSE_LOCAL -q "select * from file('$CURDIR/data_parquet/02588_data.parquet') where exchange_ts = 1670964058771367936" +$CLICKHOUSE_LOCAL -q "select exchange_ts from file('$CURDIR/data_parquet/02588_data.parquet') where exchange_ts = 1670964058771367936" +$CLICKHOUSE_LOCAL -q "select exchange_ts, market, product from file('$CURDIR/data_parquet/02588_data.parquet') where exchange_ts = 1670946478544048640" + diff --git a/tests/queries/0_stateless/02662_sparse_columns_mutations_1.reference b/tests/queries/0_stateless/02662_sparse_columns_mutations_1.reference new file mode 100644 index 00000000000..3f5c8b6ed1f --- /dev/null +++ b/tests/queries/0_stateless/02662_sparse_columns_mutations_1.reference @@ -0,0 +1,13 @@ +1_1_1_0 String Sparse +477 ['','foo'] +1_1_1_0_2 Nullable(String) Default +477 ['','foo'] +1_1_1_0_2 Nullable(String) Default +2_3_3_0 Nullable(String) Default +954 ['','foo'] +1_1_1_0_4 String Default +2_3_3_0_4 String Default +954 ['','foo'] +1_1_1_1_4 String Sparse +2_3_3_1_4 String Sparse +954 ['','foo'] diff --git a/tests/queries/0_stateless/02662_sparse_columns_mutations_1.sql b/tests/queries/0_stateless/02662_sparse_columns_mutations_1.sql new file mode 100644 index 00000000000..3bf37e8e62b --- /dev/null +++ b/tests/queries/0_stateless/02662_sparse_columns_mutations_1.sql @@ -0,0 +1,49 @@ +SET mutations_sync = 2; + +DROP TABLE IF EXISTS t_sparse_mutations_1; + +CREATE TABLE t_sparse_mutations_1 (key UInt8, id UInt64, s String) +ENGINE = MergeTree ORDER BY id PARTITION BY key +SETTINGS ratio_of_defaults_for_sparse_serialization = 0.9; + +INSERT INTO t_sparse_mutations_1 SELECT 1, number, if (number % 21 = 0, 'foo', '') FROM numbers (10000); + +SELECT name, type, serialization_kind FROM system.parts_columns +WHERE database = currentDatabase() AND table = 't_sparse_mutations_1' AND column = 's' AND active +ORDER BY name; + +SELECT countIf(s = 'foo'), arraySort(groupUniqArray(s)) FROM t_sparse_mutations_1; + +ALTER TABLE t_sparse_mutations_1 MODIFY COLUMN s Nullable(String); + +SELECT name, type, serialization_kind FROM system.parts_columns +WHERE database = currentDatabase() AND table = 't_sparse_mutations_1' AND column = 's' AND active +ORDER BY name; + +SELECT countIf(s = 'foo'), arraySort(groupUniqArray(s)) FROM t_sparse_mutations_1; + +INSERT INTO t_sparse_mutations_1 SELECT 2, number, if (number % 21 = 0, 'foo', '') FROM numbers (10000); + +SELECT name, type, serialization_kind FROM system.parts_columns +WHERE database = currentDatabase() AND table = 't_sparse_mutations_1' AND column = 's' AND active +ORDER BY name; + +SELECT countIf(s = 'foo'), arraySort(groupUniqArray(s)) FROM t_sparse_mutations_1; + +ALTER TABLE t_sparse_mutations_1 MODIFY COLUMN s String; + +SELECT name, type, serialization_kind FROM system.parts_columns +WHERE database = currentDatabase() AND table = 't_sparse_mutations_1' AND column = 's' AND active +ORDER BY name; + +SELECT countIf(s = 'foo'), arraySort(groupUniqArray(s)) FROM t_sparse_mutations_1; + +OPTIMIZE TABLE t_sparse_mutations_1 FINAL; + +SELECT name, type, serialization_kind FROM system.parts_columns +WHERE database = currentDatabase() AND table = 't_sparse_mutations_1' AND column = 's' AND active +ORDER BY name; + +SELECT countIf(s = 'foo'), arraySort(groupUniqArray(s)) FROM t_sparse_mutations_1; + +DROP TABLE t_sparse_mutations_1; diff --git a/tests/queries/0_stateless/02662_sparse_columns_mutations_2.reference b/tests/queries/0_stateless/02662_sparse_columns_mutations_2.reference new file mode 100644 index 00000000000..64eb0119982 --- /dev/null +++ b/tests/queries/0_stateless/02662_sparse_columns_mutations_2.reference @@ -0,0 +1,6 @@ +String Default +10000 49995000 +String Default +770 3848845 +String Sparse +770 3848845 diff --git a/tests/queries/0_stateless/02662_sparse_columns_mutations_2.sql b/tests/queries/0_stateless/02662_sparse_columns_mutations_2.sql new file mode 100644 index 00000000000..561bd164200 --- /dev/null +++ b/tests/queries/0_stateless/02662_sparse_columns_mutations_2.sql @@ -0,0 +1,33 @@ +SET mutations_sync = 2; + +DROP TABLE IF EXISTS t_sparse_mutations_2; + +CREATE TABLE t_sparse_mutations_2 (key UInt8, id UInt64, s String) +ENGINE = MergeTree ORDER BY id PARTITION BY key +SETTINGS ratio_of_defaults_for_sparse_serialization = 0.9; + +INSERT INTO t_sparse_mutations_2 SELECT 1, number, toString(number) FROM numbers (10000); + +SELECT type, serialization_kind FROM system.parts_columns +WHERE database = currentDatabase() AND table = 't_sparse_mutations_2' AND column = 's' AND active +ORDER BY name; + +SELECT count(), sum(s::UInt64) FROM t_sparse_mutations_2 WHERE s != ''; + +ALTER TABLE t_sparse_mutations_2 UPDATE s = '' WHERE id % 13 != 0; + +SELECT type, serialization_kind FROM system.parts_columns +WHERE database = currentDatabase() AND table = 't_sparse_mutations_2' AND column = 's' AND active +ORDER BY name; + +SELECT count(), sum(s::UInt64) FROM t_sparse_mutations_2 WHERE s != ''; + +OPTIMIZE TABLE t_sparse_mutations_2 FINAL; + +SELECT type, serialization_kind FROM system.parts_columns +WHERE database = currentDatabase() AND table = 't_sparse_mutations_2' AND column = 's' AND active +ORDER BY name; + +SELECT count(), sum(s::UInt64) FROM t_sparse_mutations_2 WHERE s != ''; + +DROP TABLE t_sparse_mutations_2; diff --git a/tests/queries/0_stateless/02662_sparse_columns_mutations_3.reference b/tests/queries/0_stateless/02662_sparse_columns_mutations_3.reference new file mode 100644 index 00000000000..1501fd27fd5 --- /dev/null +++ b/tests/queries/0_stateless/02662_sparse_columns_mutations_3.reference @@ -0,0 +1,11 @@ +String Default +Tuple(UInt64, UInt64, String, String, String) Default ['1','2','3','4','5'] ['UInt64','UInt64','String','String','String'] ['Default','Default','Default','Default','Default'] +10000 0 ['1'] ['0'] [''] +Tuple(UInt64, UInt64, String, String, String) Default ['1','2','3','4','5'] ['UInt64','UInt64','String','String','String'] ['Default','Sparse','Default','Default','Sparse'] +10000 0 ['1'] ['0'] [''] +Tuple(UInt64, UInt64, UInt64, UInt64, String) Default ['1','2','3','4','5'] ['UInt64','UInt64','UInt64','UInt64','String'] ['Default','Sparse','Default','Default','Sparse'] +10000 0 10000 0 [''] +Tuple(UInt64, UInt64, UInt64, UInt64, String) Default ['1','2','3','4','5'] ['UInt64','UInt64','UInt64','UInt64','String'] ['Default','Sparse','Default','Sparse','Sparse'] +10000 0 10000 0 [''] +Tuple(Nullable(UInt64), Nullable(UInt64), Nullable(UInt64), Nullable(UInt64), Nullable(String)) Default ['1','1.null','2','2.null','3','3.null','4','4.null','5','5.null'] ['Nullable(UInt64)','UInt8','Nullable(UInt64)','UInt8','Nullable(UInt64)','UInt8','Nullable(UInt64)','UInt8','Nullable(String)','UInt8'] ['Default','Default','Default','Default','Default','Default','Default','Default','Default','Default'] +10000 0 10000 0 [''] diff --git a/tests/queries/0_stateless/02662_sparse_columns_mutations_3.sql b/tests/queries/0_stateless/02662_sparse_columns_mutations_3.sql new file mode 100644 index 00000000000..6e66336dcbc --- /dev/null +++ b/tests/queries/0_stateless/02662_sparse_columns_mutations_3.sql @@ -0,0 +1,85 @@ +SET mutations_sync = 2; + +DROP TABLE IF EXISTS t_sparse_mutations_3; + +CREATE TABLE t_sparse_mutations_3 (key UInt8, id UInt64, s String) +ENGINE = MergeTree ORDER BY id PARTITION BY key +SETTINGS ratio_of_defaults_for_sparse_serialization = 0.9; + +INSERT INTO t_sparse_mutations_3 SELECT 1, number, toString(tuple(1, 0, '1', '0', '')) FROM numbers (10000); + +SELECT type, serialization_kind FROM system.parts_columns +WHERE database = currentDatabase() AND table = 't_sparse_mutations_3' AND column = 's' AND active +ORDER BY name; + +ALTER TABLE t_sparse_mutations_3 MODIFY COLUMN s Tuple(UInt64, UInt64, String, String, String); + +SELECT + type, + serialization_kind, + subcolumns.names, + subcolumns.types, + subcolumns.serializations +FROM system.parts_columns +WHERE database = currentDatabase() AND table = 't_sparse_mutations_3' AND column = 's' AND active +ORDER BY name; + +SELECT sum(s.1), sum(s.2), groupUniqArray(s.3), groupUniqArray(s.4), groupUniqArray(s.5) FROM t_sparse_mutations_3; + +OPTIMIZE TABLE t_sparse_mutations_3 FINAL; + +SELECT + type, + serialization_kind, + subcolumns.names, + subcolumns.types, + subcolumns.serializations +FROM system.parts_columns +WHERE database = currentDatabase() AND table = 't_sparse_mutations_3' AND column = 's' AND active +ORDER BY name; + +SELECT sum(s.1), sum(s.2), groupUniqArray(s.3), groupUniqArray(s.4), groupUniqArray(s.5) FROM t_sparse_mutations_3; + +ALTER TABLE t_sparse_mutations_3 MODIFY COLUMN s Tuple(UInt64, UInt64, UInt64, UInt64, String); + +SELECT + type, + serialization_kind, + subcolumns.names, + subcolumns.types, + subcolumns.serializations +FROM system.parts_columns +WHERE database = currentDatabase() AND table = 't_sparse_mutations_3' AND column = 's' AND active +ORDER BY name; + +SELECT sum(s.1), sum(s.2), sum(s.3), sum(s.4), groupUniqArray(s.5) FROM t_sparse_mutations_3; + +OPTIMIZE TABLE t_sparse_mutations_3 FINAL; + +SELECT + type, + serialization_kind, + subcolumns.names, + subcolumns.types, + subcolumns.serializations +FROM system.parts_columns +WHERE database = currentDatabase() AND table = 't_sparse_mutations_3' AND column = 's' AND active +ORDER BY name; + +SELECT sum(s.1), sum(s.2), sum(s.3), sum(s.4), groupUniqArray(s.5) FROM t_sparse_mutations_3; + +ALTER TABLE t_sparse_mutations_3 MODIFY COLUMN s Tuple(Nullable(UInt64), Nullable(UInt64), Nullable(UInt64), Nullable(UInt64), Nullable(String)); + +SELECT + type, + serialization_kind, + subcolumns.names, + subcolumns.types, + subcolumns.serializations +FROM system.parts_columns +WHERE database = currentDatabase() AND table = 't_sparse_mutations_3' AND column = 's' AND active +ORDER BY name; + +SELECT sum(s.1), sum(s.2), sum(s.3), sum(s.4), groupUniqArray(s.5) FROM t_sparse_mutations_3; + +DROP TABLE t_sparse_mutations_3; diff --git a/tests/queries/0_stateless/02662_sparse_columns_mutations_4.reference b/tests/queries/0_stateless/02662_sparse_columns_mutations_4.reference new file mode 100644 index 00000000000..2e24ab44f9a --- /dev/null +++ b/tests/queries/0_stateless/02662_sparse_columns_mutations_4.reference @@ -0,0 +1,2 @@ +UInt64 Sparse +String Default diff --git a/tests/queries/0_stateless/02662_sparse_columns_mutations_4.sql b/tests/queries/0_stateless/02662_sparse_columns_mutations_4.sql new file mode 100644 index 00000000000..039af658489 --- /dev/null +++ b/tests/queries/0_stateless/02662_sparse_columns_mutations_4.sql @@ -0,0 +1,21 @@ +SET mutations_sync = 2; + +DROP TABLE IF EXISTS t_sparse_mutations_4; + +CREATE TABLE t_sparse_mutations_4 (k UInt64, v UInt64) +ENGINE = MergeTree ORDER BY k +SETTINGS ratio_of_defaults_for_sparse_serialization = 0.9; + +INSERT INTO t_sparse_mutations_4 SELECT number, 0 FROM numbers(10000); + +SELECT type, serialization_kind FROM system.parts_columns +WHERE database = currentDatabase() AND table = 't_sparse_mutations_4' AND column = 'v' AND active +ORDER BY name; + +ALTER TABLE t_sparse_mutations_4 MODIFY COLUMN v String; + +SELECT type, serialization_kind FROM system.parts_columns +WHERE database = currentDatabase() AND table = 't_sparse_mutations_4' AND column = 'v' AND active +ORDER BY name; + +DROP TABLE t_sparse_mutations_4; diff --git a/tests/queries/0_stateless/02662_sparse_columns_mutations_5.reference b/tests/queries/0_stateless/02662_sparse_columns_mutations_5.reference new file mode 100644 index 00000000000..698d61cbb24 --- /dev/null +++ b/tests/queries/0_stateless/02662_sparse_columns_mutations_5.reference @@ -0,0 +1,2 @@ +Tuple(UInt64, UInt64) Default ['1','2'] ['UInt64','UInt64'] ['Sparse','Sparse'] +Tuple(UInt64, String) Default ['1','2'] ['UInt64','String'] ['Sparse','Default'] diff --git a/tests/queries/0_stateless/02662_sparse_columns_mutations_5.sql b/tests/queries/0_stateless/02662_sparse_columns_mutations_5.sql new file mode 100644 index 00000000000..79bac836bdc --- /dev/null +++ b/tests/queries/0_stateless/02662_sparse_columns_mutations_5.sql @@ -0,0 +1,21 @@ +SET mutations_sync = 2; + +DROP TABLE IF EXISTS t_sparse_mutations_5; + +CREATE TABLE t_sparse_mutations_5 (k UInt64, t Tuple(UInt64, UInt64)) +ENGINE = MergeTree ORDER BY k +SETTINGS ratio_of_defaults_for_sparse_serialization = 0.9; + +INSERT INTO t_sparse_mutations_5 SELECT number, (0, 0) FROM numbers(10000); + +SELECT type, serialization_kind, subcolumns.names, subcolumns.types, subcolumns.serializations FROM system.parts_columns +WHERE database = currentDatabase() AND table = 't_sparse_mutations_5' AND column = 't' AND active +ORDER BY name; + +ALTER TABLE t_sparse_mutations_5 MODIFY COLUMN t Tuple(UInt64, String); + +SELECT type, serialization_kind, subcolumns.names, subcolumns.types, subcolumns.serializations FROM system.parts_columns +WHERE database = currentDatabase() AND table = 't_sparse_mutations_5' AND column = 't' AND active +ORDER BY name; + +DROP TABLE t_sparse_mutations_5; diff --git a/tests/queries/0_stateless/02668_logical_optimizer_removing_redundant_checks.reference b/tests/queries/0_stateless/02668_logical_optimizer_removing_redundant_checks.reference new file mode 100644 index 00000000000..d083e178586 --- /dev/null +++ b/tests/queries/0_stateless/02668_logical_optimizer_removing_redundant_checks.reference @@ -0,0 +1,89 @@ +1 test +3 another +QUERY id: 0 + PROJECTION COLUMNS + a Int32 + b LowCardinality(String) + PROJECTION + LIST id: 1, nodes: 2 + COLUMN id: 2, column_name: a, result_type: Int32, source_id: 3 + COLUMN id: 4, column_name: b, result_type: LowCardinality(String), source_id: 3 + JOIN TREE + TABLE id: 3, table_name: default.02668_logical_optimizer + WHERE + FUNCTION id: 5, function_name: in, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 6, nodes: 2 + COLUMN id: 7, column_name: a, result_type: Int32, source_id: 3 + CONSTANT id: 8, constant_value: Tuple_(UInt64_1, UInt64_3), constant_value_type: Tuple(UInt8, UInt8) +1 test +QUERY id: 0 + PROJECTION COLUMNS + a Int32 + b LowCardinality(String) + PROJECTION + LIST id: 1, nodes: 2 + COLUMN id: 2, column_name: a, result_type: Int32, source_id: 3 + COLUMN id: 4, column_name: b, result_type: LowCardinality(String), source_id: 3 + JOIN TREE + TABLE id: 3, table_name: default.02668_logical_optimizer + WHERE + FUNCTION id: 5, function_name: equals, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 6, nodes: 2 + COLUMN id: 7, column_name: a, result_type: Int32, source_id: 3 + CONSTANT id: 8, constant_value: UInt64_1, constant_value_type: UInt8 +QUERY id: 0 + PROJECTION COLUMNS + a Int32 + b LowCardinality(String) + PROJECTION + LIST id: 1, nodes: 2 + COLUMN id: 2, column_name: a, result_type: Int32, source_id: 3 + COLUMN id: 4, column_name: b, result_type: LowCardinality(String), source_id: 3 + JOIN TREE + TABLE id: 3, table_name: default.02668_logical_optimizer + WHERE + CONSTANT id: 5, constant_value: UInt64_0, constant_value_type: UInt8 +3 another +QUERY id: 0 + PROJECTION COLUMNS + a Int32 + b LowCardinality(String) + PROJECTION + LIST id: 1, nodes: 2 + COLUMN id: 2, column_name: a, result_type: Int32, source_id: 3 + COLUMN id: 4, column_name: b, result_type: LowCardinality(String), source_id: 3 + JOIN TREE + TABLE id: 3, table_name: default.02668_logical_optimizer + WHERE + FUNCTION id: 5, function_name: and, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 6, nodes: 2 + FUNCTION id: 7, function_name: equals, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 8, nodes: 2 + COLUMN id: 9, column_name: a, result_type: Int32, source_id: 3 + CONSTANT id: 10, constant_value: UInt64_3, constant_value_type: UInt8 + FUNCTION id: 11, function_name: equals, function_type: ordinary, result_type: LowCardinality(UInt8) + ARGUMENTS + LIST id: 12, nodes: 2 + COLUMN id: 13, column_name: b, result_type: LowCardinality(String), source_id: 3 + CONSTANT id: 14, constant_value: \'another\', constant_value_type: String +2 test2 +QUERY id: 0 + PROJECTION COLUMNS + a Int32 + b LowCardinality(String) + PROJECTION + LIST id: 1, nodes: 2 + COLUMN id: 2, column_name: a, result_type: Int32, source_id: 3 + COLUMN id: 4, column_name: b, result_type: LowCardinality(String), source_id: 3 + JOIN TREE + TABLE id: 3, table_name: default.02668_logical_optimizer + WHERE + FUNCTION id: 5, function_name: equals, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 6, nodes: 2 + COLUMN id: 7, column_name: a, result_type: Int32, source_id: 3 + CONSTANT id: 8, constant_value: UInt64_2, constant_value_type: UInt8 diff --git a/tests/queries/0_stateless/02668_logical_optimizer_removing_redundant_checks.sql b/tests/queries/0_stateless/02668_logical_optimizer_removing_redundant_checks.sql new file mode 100644 index 00000000000..f20ef412215 --- /dev/null +++ b/tests/queries/0_stateless/02668_logical_optimizer_removing_redundant_checks.sql @@ -0,0 +1,26 @@ +SET allow_experimental_analyzer = 1; + +DROP TABLE IF EXISTS 02668_logical_optimizer; + +CREATE TABLE 02668_logical_optimizer +(a Int32, b LowCardinality(String)) +ENGINE=Memory; + +INSERT INTO 02668_logical_optimizer VALUES (1, 'test'), (2, 'test2'), (3, 'another'); + +SET optimize_min_equality_disjunction_chain_length = 2; + +SELECT * FROM 02668_logical_optimizer WHERE a = 1 OR 3 = a OR 1 = a; +EXPLAIN QUERY TREE SELECT * FROM 02668_logical_optimizer WHERE a = 1 OR 3 = a OR 1 = a; + +SELECT * FROM 02668_logical_optimizer WHERE a = 1 OR 1 = a; +EXPLAIN QUERY TREE SELECT * FROM 02668_logical_optimizer WHERE a = 1 OR 1 = a; + +SELECT * FROM 02668_logical_optimizer WHERE a = 1 AND 2 = a; +EXPLAIN QUERY TREE SELECT * FROM 02668_logical_optimizer WHERE a = 1 AND 2 = a; + +SELECT * FROM 02668_logical_optimizer WHERE 3 = a AND b = 'another' AND a = 3; +EXPLAIN QUERY TREE SELECT * FROM 02668_logical_optimizer WHERE a = 3 AND b = 'another' AND a = 3; + +SELECT * FROM 02668_logical_optimizer WHERE a = 2 AND 2 = a; +EXPLAIN QUERY TREE SELECT * FROM 02668_logical_optimizer WHERE a = 2 AND 2 = a; diff --git a/tests/queries/0_stateless/02669_alter_modify_to_nullable.reference b/tests/queries/0_stateless/02669_alter_modify_to_nullable.reference new file mode 100644 index 00000000000..aff80e1d699 --- /dev/null +++ b/tests/queries/0_stateless/02669_alter_modify_to_nullable.reference @@ -0,0 +1,8 @@ +1_1_1_0 String Default +2_2_2_0 String Sparse +20000 10435 ['','bar','foo'] +1_1_1_0_3 String Default +2_2_2_0_3 Nullable(String) Default +20000 10435 ['','bar','foo'] +1_1_1_0_3 0 +2_2_2_0_3 10000 diff --git a/tests/queries/0_stateless/02669_alter_modify_to_nullable.sql b/tests/queries/0_stateless/02669_alter_modify_to_nullable.sql new file mode 100644 index 00000000000..862280fd7cd --- /dev/null +++ b/tests/queries/0_stateless/02669_alter_modify_to_nullable.sql @@ -0,0 +1,31 @@ +DROP TABLE IF EXISTS t_modify_to_nullable; + +CREATE TABLE t_modify_to_nullable (key UInt64, id UInt64, s String) +ENGINE = MergeTree ORDER BY id PARTITION BY key +SETTINGS min_bytes_for_wide_part = 0, ratio_of_defaults_for_sparse_serialization = 0.9; + +INSERT INTO t_modify_to_nullable SELECT 1, number, 'foo' FROM numbers(10000); +INSERT INTO t_modify_to_nullable SELECT 2, number, if (number % 23 = 0, 'bar', '') FROM numbers(10000); + +SELECT name, type, serialization_kind FROM system.parts_columns +WHERE database = currentDatabase() AND table = 't_modify_to_nullable' AND column = 's' AND active +ORDER BY name; + +SELECT count(s), countIf(s != ''), arraySort(groupUniqArray(s)) FROM t_modify_to_nullable; + +SET mutations_sync = 2; +ALTER TABLE t_modify_to_nullable MODIFY COLUMN s Nullable(String); + +SELECT name, type, serialization_kind FROM system.parts_columns +WHERE database = currentDatabase() AND table = 't_modify_to_nullable' AND column = 's' AND active +ORDER BY name; + +SELECT count(s), countIf(s != ''), arraySort(groupUniqArray(s)) FROM t_modify_to_nullable; + +SYSTEM FLUSH LOGS; + +SELECT part_name, read_rows FROM system.part_log +WHERE database = currentDatabase() AND table = 't_modify_to_nullable' AND event_type = 'MutatePart' +ORDER BY part_name; + +DROP TABLE t_modify_to_nullable; diff --git a/tests/queries/0_stateless/02670_constant_skip_index.reference b/tests/queries/0_stateless/02670_constant_skip_index.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02670_constant_skip_index.sql b/tests/queries/0_stateless/02670_constant_skip_index.sql new file mode 100644 index 00000000000..97dd2ab33c9 --- /dev/null +++ b/tests/queries/0_stateless/02670_constant_skip_index.sql @@ -0,0 +1,25 @@ + +DROP TABLE IF EXISTS t_constant_index; + +CREATE TABLE t_constant_index +( + id UInt64, + INDEX t_constant_index 'foo' TYPE set(2) GRANULARITY 1 +) ENGINE = MergeTree +ORDER BY id; -- { serverError INCORRECT_QUERY } + +CREATE TABLE t_constant_index +( + id UInt64, + INDEX t_constant_index id + rand() TYPE set(2) GRANULARITY 1 +) ENGINE = MergeTree +ORDER BY id; -- { serverError BAD_ARGUMENTS } + +CREATE TABLE t_constant_index +( + id UInt64, + INDEX t_constant_index id * 2 TYPE set(2) GRANULARITY 1 +) ENGINE = MergeTree +ORDER BY id; + +DROP TABLE t_constant_index; diff --git a/tests/queries/0_stateless/02674_date_int_string_json_inference.reference b/tests/queries/0_stateless/02674_date_int_string_json_inference.reference new file mode 100644 index 00000000000..2e89d6a15a4 --- /dev/null +++ b/tests/queries/0_stateless/02674_date_int_string_json_inference.reference @@ -0,0 +1 @@ +x Nullable(String) diff --git a/tests/queries/0_stateless/02674_date_int_string_json_inference.sql b/tests/queries/0_stateless/02674_date_int_string_json_inference.sql new file mode 100644 index 00000000000..21abf763cbf --- /dev/null +++ b/tests/queries/0_stateless/02674_date_int_string_json_inference.sql @@ -0,0 +1,2 @@ +desc format(JSONEachRow, '{"x" : "2020-01-01"}, {"x" : "1000"}') + diff --git a/tests/queries/0_stateless/02674_trivial_count_analyzer.reference b/tests/queries/0_stateless/02674_trivial_count_analyzer.reference new file mode 100644 index 00000000000..05feadb58a0 --- /dev/null +++ b/tests/queries/0_stateless/02674_trivial_count_analyzer.reference @@ -0,0 +1,47 @@ +-- { echoOn } +set allow_experimental_analyzer=1; +set optimize_trivial_count_query=1; +create table m3(a Int64, b UInt64) Engine=MergeTree order by tuple(); +select count() from m3; +0 +insert into m3 values (0,0); +insert into m3 values (-1,1); +select trimBoth(explain) from (explain select count() from m3) where explain like '%ReadFromPreparedSource (Optimized trivial count)%'; +ReadFromPreparedSource (Optimized trivial count) +select count() from m3; +2 +select count(*) from m3; +2 +select count(a) from m3; +2 +select count(b) from m3; +2 +select count() + 1 from m3; +3 +drop table m3; +-- checking queries with FINAL +create table replacing_m3(a Int64, b UInt64) Engine=ReplacingMergeTree() order by (a, b); +SYSTEM STOP MERGES replacing_m3; +select count() from replacing_m3; +0 +insert into replacing_m3 values (0,0); +insert into replacing_m3 values (0,0); +insert into replacing_m3 values (-1,1); +insert into replacing_m3 values (-2,2); +select trimBoth(explain) from (explain select count() from replacing_m3) where explain like '%ReadFromPreparedSource (Optimized trivial count)%'; +ReadFromPreparedSource (Optimized trivial count) +select count() from replacing_m3; +4 +select count(*) from replacing_m3; +4 +select count(a) from replacing_m3; +4 +select count(b) from replacing_m3; +4 +select count() from replacing_m3 FINAL; +3 +select count(a) from replacing_m3 FINAL; +3 +select count(b) from replacing_m3 FINAL; +3 +drop table replacing_m3; diff --git a/tests/queries/0_stateless/02674_trivial_count_analyzer.sql b/tests/queries/0_stateless/02674_trivial_count_analyzer.sql new file mode 100644 index 00000000000..988d1b9ba92 --- /dev/null +++ b/tests/queries/0_stateless/02674_trivial_count_analyzer.sql @@ -0,0 +1,45 @@ +drop table if exists m3; +drop table if exists replacing_m3; + +-- { echoOn } +set allow_experimental_analyzer=1; +set optimize_trivial_count_query=1; + +create table m3(a Int64, b UInt64) Engine=MergeTree order by tuple(); + +select count() from m3; + +insert into m3 values (0,0); +insert into m3 values (-1,1); + +select trimBoth(explain) from (explain select count() from m3) where explain like '%ReadFromPreparedSource (Optimized trivial count)%'; +select count() from m3; +select count(*) from m3; +select count(a) from m3; +select count(b) from m3; +select count() + 1 from m3; + +drop table m3; + +-- checking queries with FINAL +create table replacing_m3(a Int64, b UInt64) Engine=ReplacingMergeTree() order by (a, b); +SYSTEM STOP MERGES replacing_m3; + +select count() from replacing_m3; + +insert into replacing_m3 values (0,0); +insert into replacing_m3 values (0,0); +insert into replacing_m3 values (-1,1); +insert into replacing_m3 values (-2,2); + +select trimBoth(explain) from (explain select count() from replacing_m3) where explain like '%ReadFromPreparedSource (Optimized trivial count)%'; +select count() from replacing_m3; +select count(*) from replacing_m3; +select count(a) from replacing_m3; +select count(b) from replacing_m3; + +select count() from replacing_m3 FINAL; +select count(a) from replacing_m3 FINAL; +select count(b) from replacing_m3 FINAL; + +drop table replacing_m3; diff --git a/tests/queries/0_stateless/02675_predicate_push_down_filled_join_fix.reference b/tests/queries/0_stateless/02675_predicate_push_down_filled_join_fix.reference new file mode 100644 index 00000000000..ecdb62c5cb5 --- /dev/null +++ b/tests/queries/0_stateless/02675_predicate_push_down_filled_join_fix.reference @@ -0,0 +1,33 @@ +Expression ((Project names + (Projection + ))) +Header: t1.id UInt64 + t1.value String + t2.value String +Actions: INPUT : 0 -> t1.id_0 UInt64 : 0 + INPUT : 1 -> t1.value_1 String : 1 + INPUT : 2 -> t2.value_2 String : 2 + ALIAS t1.id_0 :: 0 -> t1.id UInt64 : 3 + ALIAS t1.value_1 :: 1 -> t1.value String : 0 + ALIAS t2.value_2 :: 2 -> t2.value String : 1 +Positions: 3 0 1 + FilledJoin (Filled JOIN) + Header: t1.id_0 UInt64 + t1.value_1 String + t2.value_2 String + Filter (( + (JOIN actions + Change column names to column identifiers))) + Header: t1.id_0 UInt64 + t1.value_1 String + Filter column: equals(t1.id_0, 0_UInt8) (removed) + Actions: INPUT : 0 -> id UInt64 : 0 + INPUT : 1 -> value String : 1 + COLUMN Const(UInt8) -> 0_UInt8 UInt8 : 2 + ALIAS id :: 0 -> t1.id_0 UInt64 : 3 + ALIAS value :: 1 -> t1.value_1 String : 0 + FUNCTION equals(t1.id_0 : 3, 0_UInt8 :: 2) -> equals(t1.id_0, 0_UInt8) UInt8 : 1 + Positions: 1 3 0 + ReadFromMergeTree (default.test_table) + Header: id UInt64 + value String + ReadType: Default + Parts: 1 + Granules: 1 +0 Value JoinValue diff --git a/tests/queries/0_stateless/02675_predicate_push_down_filled_join_fix.sql b/tests/queries/0_stateless/02675_predicate_push_down_filled_join_fix.sql new file mode 100644 index 00000000000..73baad11634 --- /dev/null +++ b/tests/queries/0_stateless/02675_predicate_push_down_filled_join_fix.sql @@ -0,0 +1,27 @@ +SET allow_experimental_analyzer = 1; +SET single_join_prefer_left_table = 0; + +DROP TABLE IF EXISTS test_table; +CREATE TABLE test_table +( + id UInt64, + value String +) ENGINE=MergeTree ORDER BY id; + +INSERT INTO test_table VALUES (0, 'Value'); + +DROP TABLE IF EXISTS test_table_join; +CREATE TABLE test_table_join +( + id UInt64, + value String +) ENGINE = Join(All, inner, id); + +INSERT INTO test_table_join VALUES (0, 'JoinValue'); + +EXPLAIN header = 1, actions = 1 SELECT t1.id, t1.value, t2.value FROM test_table AS t1 INNER JOIN test_table_join AS t2 ON t1.id = t2.id WHERE t1.id = 0; + +SELECT t1.id, t1.value, t2.value FROM test_table AS t1 INNER JOIN test_table_join AS t2 ON t1.id = t2.id WHERE t1.id = 0; + +DROP TABLE test_table_join; +DROP TABLE test_table; diff --git a/tests/queries/0_stateless/02675_replicated_merge_tree_insert_zookeeper_long.reference b/tests/queries/0_stateless/02675_replicated_merge_tree_insert_zookeeper_long.reference new file mode 100644 index 00000000000..0cfbf08886f --- /dev/null +++ b/tests/queries/0_stateless/02675_replicated_merge_tree_insert_zookeeper_long.reference @@ -0,0 +1 @@ +2 diff --git a/tests/queries/0_stateless/02675_replicated_merge_tree_insert_zookeeper_long.sql b/tests/queries/0_stateless/02675_replicated_merge_tree_insert_zookeeper_long.sql new file mode 100644 index 00000000000..194ea9bfcc1 --- /dev/null +++ b/tests/queries/0_stateless/02675_replicated_merge_tree_insert_zookeeper_long.sql @@ -0,0 +1,15 @@ +-- Tags: no-s3-storage + +DROP TABLE IF EXISTS inmemory_test; + +CREATE TABLE inmemory_test (d Date, id String) +ENGINE=ReplicatedMergeTree('/clickhouse/tables/{database}/inmemory_test', 'r1') +PARTITION BY toYYYYMMDD(d) ORDER BY (d, id) +SETTINGS min_rows_for_compact_part = 10, index_granularity = 8192; + +INSERT INTO inmemory_test(d, id) VALUES('2023-01-01', 'abcdefghijklmnopqrstuvwxyz'); +INSERT INTO inmemory_test(d, id) VALUES('2023-01-01', 'a1234567890123456789012345'); + +SELECT COUNT(1) FROM inmemory_test; + +DROP TABLE inmemory_test; diff --git a/tests/queries/0_stateless/02675_sparse_columns_clear_column.reference b/tests/queries/0_stateless/02675_sparse_columns_clear_column.reference new file mode 100644 index 00000000000..56fa4a9ebea --- /dev/null +++ b/tests/queries/0_stateless/02675_sparse_columns_clear_column.reference @@ -0,0 +1,6 @@ +arr Default +v Sparse +arr Default +arr Default +v Sparse +0 [] diff --git a/tests/queries/0_stateless/02675_sparse_columns_clear_column.sql b/tests/queries/0_stateless/02675_sparse_columns_clear_column.sql new file mode 100644 index 00000000000..781030ef7b4 --- /dev/null +++ b/tests/queries/0_stateless/02675_sparse_columns_clear_column.sql @@ -0,0 +1,34 @@ +DROP TABLE IF EXISTS t_sparse_columns_clear; + +CREATE TABLE t_sparse_columns_clear (arr Array(UInt64), v UInt64) +ENGINE = MergeTree ORDER BY tuple() +SETTINGS + ratio_of_defaults_for_sparse_serialization = 0.9, + min_bytes_for_wide_part=0; + +INSERT INTO t_sparse_columns_clear SELECT [number], 0 FROM numbers(1000); + +SELECT column, serialization_kind FROM system.parts_columns +WHERE database = currentDatabase() AND table = 't_sparse_columns_clear' AND active +ORDER BY column; + +SET mutations_sync = 2; +SET alter_sync = 2; + +ALTER TABLE t_sparse_columns_clear CLEAR COLUMN v; + +SELECT column, serialization_kind FROM system.parts_columns +WHERE database = currentDatabase() AND table = 't_sparse_columns_clear' AND active +ORDER BY column; + +OPTIMIZE TABLE t_sparse_columns_clear FINAL; + +SELECT column, serialization_kind FROM system.parts_columns +WHERE database = currentDatabase() AND table = 't_sparse_columns_clear' AND active +ORDER BY column; + +DROP TABLE t_sparse_columns_clear SYNC; + +SYSTEM FLUSH LOGS; + +SELECT count(), groupArray(message) FROM system.text_log WHERE logger_name LIKE '%' || currentDatabase() || '.t_sparse_columns_clear' || '%' AND level = 'Error'; diff --git a/tests/queries/0_stateless/02676_analyzer_limit_offset.reference b/tests/queries/0_stateless/02676_analyzer_limit_offset.reference new file mode 100644 index 00000000000..96483268d43 --- /dev/null +++ b/tests/queries/0_stateless/02676_analyzer_limit_offset.reference @@ -0,0 +1,63 @@ +0 +1 +2 +3 +4 +15 +15 +16 +16 +17 +30 +30 +31 +31 +32 +102 +103 +104 +105 +105 +106 +107 +108 +109 +105 +106 +107 +108 +109 +60 +60 +61 +61 +62 +62 +63 +63 +64 +64 +60 +35 +35 +36 +36 +37 +37 +38 +38 +39 +39 +105 +106 +107 +108 +109 +12 +13 +13 +14 +14 +15 +15 +16 diff --git a/tests/queries/0_stateless/02676_analyzer_limit_offset.sql b/tests/queries/0_stateless/02676_analyzer_limit_offset.sql new file mode 100644 index 00000000000..39c6b85f088 --- /dev/null +++ b/tests/queries/0_stateless/02676_analyzer_limit_offset.sql @@ -0,0 +1,34 @@ +set allow_experimental_analyzer=1; + +DROP TABLE IF EXISTS test; +CREATE TABLE test (i UInt64) Engine = MergeTree() order by i; +INSERT INTO test SELECT number FROM numbers(100); +INSERT INTO test SELECT number FROM numbers(10,100); +OPTIMIZE TABLE test FINAL; + +-- Only set limit +SET limit = 5; +SELECT * FROM test; -- 5 rows +SELECT * FROM test OFFSET 20; -- 5 rows +SELECT * FROM (SELECT i FROM test LIMIT 10 OFFSET 50) TMP; -- 5 rows +SELECT * FROM test LIMIT 4 OFFSET 192; -- 4 rows +SELECT * FROM test LIMIT 10 OFFSET 195; -- 5 rows + +-- Only set offset +SET limit = 0; +SET offset = 195; +SELECT * FROM test; -- 5 rows +SELECT * FROM test OFFSET 20; -- no result +SELECT * FROM test LIMIT 100; -- no result +SET offset = 10; +SELECT * FROM test LIMIT 20 OFFSET 100; -- 10 rows +SELECT * FROM test LIMIT 11 OFFSET 100; -- 1 rows + +-- offset and limit together +SET limit = 10; +SELECT * FROM test LIMIT 50 OFFSET 50; -- 10 rows +SELECT * FROM test LIMIT 50 OFFSET 190; -- 0 rows +SELECT * FROM test LIMIT 50 OFFSET 185; -- 5 rows +SELECT * FROM test LIMIT 18 OFFSET 5; -- 8 rows + +DROP TABLE test; diff --git a/tests/queries/0_stateless/02676_distinct_reading_in_order_analyzer.reference b/tests/queries/0_stateless/02676_distinct_reading_in_order_analyzer.reference new file mode 100644 index 00000000000..016202cfb66 --- /dev/null +++ b/tests/queries/0_stateless/02676_distinct_reading_in_order_analyzer.reference @@ -0,0 +1 @@ +MergeTreeInOrder diff --git a/tests/queries/0_stateless/02676_distinct_reading_in_order_analyzer.sql b/tests/queries/0_stateless/02676_distinct_reading_in_order_analyzer.sql new file mode 100644 index 00000000000..f00c1322e1d --- /dev/null +++ b/tests/queries/0_stateless/02676_distinct_reading_in_order_analyzer.sql @@ -0,0 +1,8 @@ +drop table if exists t; + +set allow_experimental_analyzer=1; + +create table t (a UInt64, b UInt64) engine=MergeTree() order by (a); +insert into t select number % 2, number from numbers(10); + +select splitByChar(' ', trimBoth(explain))[1] from (explain pipeline select distinct a from t) where explain like '%MergeTreeInOrder%'; diff --git a/tests/queries/0_stateless/02677_analyzer_bitmap_has_any.reference b/tests/queries/0_stateless/02677_analyzer_bitmap_has_any.reference new file mode 100644 index 00000000000..16d7e43ecb3 --- /dev/null +++ b/tests/queries/0_stateless/02677_analyzer_bitmap_has_any.reference @@ -0,0 +1,4 @@ +1 0 +-------------- +-------------- +1 0 diff --git a/tests/queries/0_stateless/02677_analyzer_bitmap_has_any.sql b/tests/queries/0_stateless/02677_analyzer_bitmap_has_any.sql new file mode 100644 index 00000000000..4af06634c66 --- /dev/null +++ b/tests/queries/0_stateless/02677_analyzer_bitmap_has_any.sql @@ -0,0 +1,36 @@ +SELECT + bitmapHasAny(bitmapBuild([toUInt8(1)]), ( + SELECT groupBitmapState(toUInt8(1)) + )) has1, + bitmapHasAny(bitmapBuild([toUInt64(1)]), ( + SELECT groupBitmapState(toUInt64(2)) + )) has2; + +SELECT '--------------'; + +SELECT * +FROM +( + SELECT + bitmapHasAny(bitmapBuild([toUInt8(1)]), ( + SELECT groupBitmapState(toUInt8(1)) + )) has1, + bitmapHasAny(bitmapBuild([toUInt64(1)]), ( + SELECT groupBitmapState(toUInt64(2)) + )) has2 +); -- { serverError 43 } + +SELECT '--------------'; + +SELECT * +FROM +( + SELECT + bitmapHasAny(bitmapBuild([toUInt8(1)]), ( + SELECT groupBitmapState(toUInt8(1)) + )) has1, + bitmapHasAny(bitmapBuild([toUInt64(1)]), ( + SELECT groupBitmapState(toUInt64(2)) + )) has2 +) SETTINGS allow_experimental_analyzer = 1; + diff --git a/tests/queries/0_stateless/02677_decode_url_component.reference b/tests/queries/0_stateless/02677_decode_url_component.reference new file mode 100644 index 00000000000..5f88856dc1c --- /dev/null +++ b/tests/queries/0_stateless/02677_decode_url_component.reference @@ -0,0 +1,2 @@ +%D0%BA%D0%BB%D0%B8%D0%BA%D1%85%D0%B0%D1%83%D1%81 1 +1 diff --git a/tests/queries/0_stateless/02677_decode_url_component.sql b/tests/queries/0_stateless/02677_decode_url_component.sql new file mode 100644 index 00000000000..68345b5de16 --- /dev/null +++ b/tests/queries/0_stateless/02677_decode_url_component.sql @@ -0,0 +1,5 @@ +SELECT + encodeURLComponent('кликхаус') AS encoded, + decodeURLComponent(encoded) = 'кликхаус' AS expected_EQ; + +SELECT DISTINCT decodeURLComponent(encodeURLComponent(randomString(100) AS x)) = x FROM numbers(100000); diff --git a/tests/queries/0_stateless/02677_grace_hash_limit_race.reference b/tests/queries/0_stateless/02677_grace_hash_limit_race.reference new file mode 100644 index 00000000000..83b33d238da --- /dev/null +++ b/tests/queries/0_stateless/02677_grace_hash_limit_race.reference @@ -0,0 +1 @@ +1000 diff --git a/tests/queries/0_stateless/02677_grace_hash_limit_race.sql b/tests/queries/0_stateless/02677_grace_hash_limit_race.sql new file mode 100644 index 00000000000..55262ab2455 --- /dev/null +++ b/tests/queries/0_stateless/02677_grace_hash_limit_race.sql @@ -0,0 +1,16 @@ +DROP TABLE IF EXISTS test_grace_hash; + +CREATE TABLE test_grace_hash (id UInt32, value UInt64) ENGINE = MergeTree ORDER BY id; + +INSERT INTO test_grace_hash SELECT number, number % 100 = 0 FROM numbers(100000); + +SET join_algorithm = 'grace_hash'; + +SELECT count() FROM ( + SELECT f.id FROM test_grace_hash AS f + LEFT JOIN test_grace_hash AS d + ON f.id = d.id + LIMIT 1000 +); + +DROP TABLE test_grace_hash; diff --git a/tests/queries/0_stateless/02678_explain_pipeline_graph_with_projection.reference b/tests/queries/0_stateless/02678_explain_pipeline_graph_with_projection.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02678_explain_pipeline_graph_with_projection.sql b/tests/queries/0_stateless/02678_explain_pipeline_graph_with_projection.sql new file mode 100644 index 00000000000..e8b7405d602 --- /dev/null +++ b/tests/queries/0_stateless/02678_explain_pipeline_graph_with_projection.sql @@ -0,0 +1,12 @@ +DROP TABLE IF EXISTS t1; +CREATE TABLE t1(ID UInt64, name String) engine=MergeTree order by ID; + +insert into t1(ID, name) values (1, 'abc'), (2, 'bbb'); + +-- The returned node order is uncertain +explain pipeline graph=1 select count(ID) from t1 FORMAT Null; +explain pipeline graph=1 select sum(1) from t1 FORMAT Null; +explain pipeline graph=1 select min(ID) from t1 FORMAT Null; +explain pipeline graph=1 select max(ID) from t1 FORMAT Null; + +DROP TABLE t1; diff --git a/tests/queries/0_stateless/02679_explain_merge_tree_prewhere_row_policy.reference b/tests/queries/0_stateless/02679_explain_merge_tree_prewhere_row_policy.reference new file mode 100644 index 00000000000..2f3b59e5530 --- /dev/null +++ b/tests/queries/0_stateless/02679_explain_merge_tree_prewhere_row_policy.reference @@ -0,0 +1,26 @@ +Expression ((Projection + Before ORDER BY)) +Header: id UInt64 + value String +Actions: INPUT :: 0 -> id UInt64 : 0 + INPUT :: 1 -> value String : 1 +Positions: 0 1 + ReadFromMergeTree (default.test_table) + Header: id UInt64 + value String + ReadType: Default + Parts: 0 + Granules: 0 + Prewhere info + Need filter: 1 + Prewhere filter + Prewhere filter column: equals(id, 5) (removed) + Actions: INPUT : 0 -> id UInt64 : 0 + COLUMN Const(UInt8) -> 5 UInt8 : 1 + FUNCTION equals(id : 0, 5 :: 1) -> equals(id, 5) UInt8 : 2 + Positions: 2 0 + Row level filter + Row level filter column: greaterOrEquals(id, 5) + Actions: INPUT : 0 -> id UInt64 : 0 + COLUMN Const(UInt8) -> 5 UInt8 : 1 + FUNCTION greaterOrEquals(id : 0, 5 :: 1) -> greaterOrEquals(id, 5) UInt8 : 2 + Positions: 2 0 diff --git a/tests/queries/0_stateless/02679_explain_merge_tree_prewhere_row_policy.sql b/tests/queries/0_stateless/02679_explain_merge_tree_prewhere_row_policy.sql new file mode 100644 index 00000000000..8099ccc0b0d --- /dev/null +++ b/tests/queries/0_stateless/02679_explain_merge_tree_prewhere_row_policy.sql @@ -0,0 +1,16 @@ +DROP TABLE IF EXISTS test_table; +CREATE TABLE test_table +( + id UInt64, + value String +) ENGINE=MergeTree ORDER BY id; + +INSERT INTO test_table VALUES (0, 'Value'); + +DROP ROW POLICY IF EXISTS test_row_policy ON test_table; +CREATE ROW POLICY test_row_policy ON test_table USING id >= 5 TO ALL; + +EXPLAIN header = 1, actions = 1 SELECT id, value FROM test_table PREWHERE id = 5; + +DROP ROW POLICY test_row_policy ON test_table; +DROP TABLE test_table; diff --git a/tests/queries/0_stateless/02679_query_parameters_dangling_pointer.reference b/tests/queries/0_stateless/02679_query_parameters_dangling_pointer.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02679_query_parameters_dangling_pointer.sql b/tests/queries/0_stateless/02679_query_parameters_dangling_pointer.sql new file mode 100644 index 00000000000..7705b860e8e --- /dev/null +++ b/tests/queries/0_stateless/02679_query_parameters_dangling_pointer.sql @@ -0,0 +1,4 @@ +-- There is no use-after-free in the following query: + +SET param_o = 'a'; +CREATE TABLE test.xxx (a Int64) ENGINE=MergeTree ORDER BY ({o:String}); -- { serverError 44 } diff --git a/tests/queries/0_stateless/02680_datetime64_monotonic_check.reference b/tests/queries/0_stateless/02680_datetime64_monotonic_check.reference new file mode 100644 index 00000000000..24d80c55377 --- /dev/null +++ b/tests/queries/0_stateless/02680_datetime64_monotonic_check.reference @@ -0,0 +1 @@ +22 0 1 diff --git a/tests/queries/0_stateless/02680_datetime64_monotonic_check.sql b/tests/queries/0_stateless/02680_datetime64_monotonic_check.sql new file mode 100644 index 00000000000..63ea7a5f639 --- /dev/null +++ b/tests/queries/0_stateless/02680_datetime64_monotonic_check.sql @@ -0,0 +1,15 @@ +DROP TABLE IF EXISTS 02680_datetime64_monotonic_check; + +CREATE TABLE 02680_datetime64_monotonic_check (`t` DateTime64(3), `x` Nullable(Decimal(18, 14))) +ENGINE = MergeTree +PARTITION BY toYYYYMMDD(t) +ORDER BY x SETTINGS allow_nullable_key = 1; + +INSERT INTO 02680_datetime64_monotonic_check VALUES (toDateTime64('2023-03-13 00:00:00', 3, 'Asia/Jerusalem'), 123); + +SELECT toHour(toTimeZone(t, 'UTC')) AS toHour_UTC, toHour(toTimeZone(t, 'Asia/Jerusalem')) AS toHour_Israel, count() +FROM 02680_datetime64_monotonic_check +WHERE toHour_Israel = 0 +GROUP BY toHour_UTC, toHour_Israel; + +DROP TABLE 02680_datetime64_monotonic_check; diff --git a/tests/queries/0_stateless/02680_instr_alias_for_position_case_insensitive.reference b/tests/queries/0_stateless/02680_instr_alias_for_position_case_insensitive.reference new file mode 100644 index 00000000000..4792e70f333 --- /dev/null +++ b/tests/queries/0_stateless/02680_instr_alias_for_position_case_insensitive.reference @@ -0,0 +1,2 @@ +2 +3 diff --git a/tests/queries/0_stateless/02680_instr_alias_for_position_case_insensitive.sql b/tests/queries/0_stateless/02680_instr_alias_for_position_case_insensitive.sql new file mode 100644 index 00000000000..c1c55c2c982 --- /dev/null +++ b/tests/queries/0_stateless/02680_instr_alias_for_position_case_insensitive.sql @@ -0,0 +1,2 @@ +select INSTR('hello', 'e'); +select INSTR('hELlo', 'L'); diff --git a/tests/queries/0_stateless/02680_lc_null_as_default.reference b/tests/queries/0_stateless/02680_lc_null_as_default.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02680_lc_null_as_default.sql b/tests/queries/0_stateless/02680_lc_null_as_default.sql new file mode 100644 index 00000000000..f6bfad37771 --- /dev/null +++ b/tests/queries/0_stateless/02680_lc_null_as_default.sql @@ -0,0 +1,6 @@ +drop table if exists test_null_as_default__fuzz_46; +SET allow_suspicious_low_cardinality_types = 1; +CREATE TABLE test_null_as_default__fuzz_46 (a Nullable(DateTime64(3)), b LowCardinality(Float32) DEFAULT a + 1000) ENGINE = Memory; +INSERT INTO test_null_as_default__fuzz_46 SELECT 1, NULL UNION ALL SELECT 2, NULL; +drop table test_null_as_default__fuzz_46; + diff --git a/tests/queries/0_stateless/01920_async_drain_connections.reference b/tests/queries/0_stateless/25340_logical_optimizer_alias_bug.reference similarity index 50% rename from tests/queries/0_stateless/01920_async_drain_connections.reference rename to tests/queries/0_stateless/25340_logical_optimizer_alias_bug.reference index aa47d0d46d4..573541ac970 100644 --- a/tests/queries/0_stateless/01920_async_drain_connections.reference +++ b/tests/queries/0_stateless/25340_logical_optimizer_alias_bug.reference @@ -1,2 +1 @@ 0 -0 diff --git a/tests/queries/0_stateless/25340_logical_optimizer_alias_bug.sql b/tests/queries/0_stateless/25340_logical_optimizer_alias_bug.sql new file mode 100644 index 00000000000..5b13eea5e4b --- /dev/null +++ b/tests/queries/0_stateless/25340_logical_optimizer_alias_bug.sql @@ -0,0 +1,2 @@ +create table test_local (id UInt32, path LowCardinality(String)) engine = MergeTree order by id; +WITH ((position(path, '/a') > 0) AND (NOT (position(path, 'a') > 0))) OR (path = '/b') OR (path = '/b/') as alias1 SELECT max(alias1) FROM remote('127.0.0.{1,2}', currentDatabase(), test_local) WHERE (id = 299386662); diff --git a/tests/queries/0_stateless/25340_storage_join_insert_select_deadlock.reference b/tests/queries/0_stateless/25340_storage_join_insert_select_deadlock.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/25340_storage_join_insert_select_deadlock.sql b/tests/queries/0_stateless/25340_storage_join_insert_select_deadlock.sql new file mode 100644 index 00000000000..59528511357 --- /dev/null +++ b/tests/queries/0_stateless/25340_storage_join_insert_select_deadlock.sql @@ -0,0 +1,16 @@ +DROP TABLE IF EXISTS test_table_join; + +CREATE TABLE test_table_join +( + id UInt64, + value String +) ENGINE = Join(Any, Left, id); + +INSERT INTO test_table_join VALUES (1, 'q'); + +INSERT INTO test_table_join SELECT * from test_table_join; -- { serverError DEADLOCK_AVOIDED } + +INSERT INTO test_table_join SELECT * FROM (SELECT 1 as id) AS t1 ANY LEFT JOIN test_table_join USING (id); -- { serverError DEADLOCK_AVOIDED } +INSERT INTO test_table_join SELECT id, toString(id) FROM (SELECT 1 as id) AS t1 ANY LEFT JOIN (SELECT id FROM test_table_join) AS t2 USING (id); -- { serverError DEADLOCK_AVOIDED } + +DROP TABLE IF EXISTS test_table_join; diff --git a/tests/queries/0_stateless/25341_inverted_idx_checksums.reference b/tests/queries/0_stateless/25341_inverted_idx_checksums.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/25341_inverted_idx_checksums.sql b/tests/queries/0_stateless/25341_inverted_idx_checksums.sql new file mode 100644 index 00000000000..92ffa7a6196 --- /dev/null +++ b/tests/queries/0_stateless/25341_inverted_idx_checksums.sql @@ -0,0 +1,16 @@ +SET allow_experimental_inverted_index = 1; + +CREATE TABLE t +( + `key` UInt64, + `str` String, + INDEX inv_idx str TYPE inverted(0) GRANULARITY 1 +) +ENGINE = MergeTree +ORDER BY key; + +INSERT INTO t VALUES (1, 'Hello World'); + +ALTER TABLE t DETACH PART 'all_1_1_0'; + +ALTER TABLE t ATTACH PART 'all_1_1_0'; \ No newline at end of file diff --git a/tests/queries/0_stateless/25400_marked_dropped_tables.reference b/tests/queries/0_stateless/25400_marked_dropped_tables.reference new file mode 100644 index 00000000000..6fc5caff0cb --- /dev/null +++ b/tests/queries/0_stateless/25400_marked_dropped_tables.reference @@ -0,0 +1,8 @@ +25400_marked_dropped_tables MergeTree +index UInt32 +database String +table String +uuid UUID +engine String +metadata_dropped_path String +table_dropped_time DateTime diff --git a/tests/queries/0_stateless/25400_marked_dropped_tables.sql b/tests/queries/0_stateless/25400_marked_dropped_tables.sql new file mode 100644 index 00000000000..101642fa779 --- /dev/null +++ b/tests/queries/0_stateless/25400_marked_dropped_tables.sql @@ -0,0 +1,11 @@ +-- Tags: no-ordinary-database + +SET database_atomic_wait_for_drop_and_detach_synchronously = 0; +DROP TABLE IF EXISTS 25400_marked_dropped_tables; + +CREATE TABLE 25400_marked_dropped_tables (id Int32) Engine=MergeTree() ORDER BY id; +DROP TABLE 25400_marked_dropped_tables; + +SELECT table, engine FROM system.marked_dropped_tables WHERE database = currentDatabase() LIMIT 1; +DESCRIBE TABLE system.marked_dropped_tables; + diff --git a/tests/queries/0_stateless/data_parquet/02588_data.parquet b/tests/queries/0_stateless/data_parquet/02588_data.parquet new file mode 100644 index 00000000000..e00b869233d Binary files /dev/null and b/tests/queries/0_stateless/data_parquet/02588_data.parquet differ diff --git a/tests/queries/0_stateless/format_schemas/02566_ipv4_ipv6.capnp b/tests/queries/0_stateless/format_schemas/02566_ipv4_ipv6.capnp new file mode 100644 index 00000000000..f999043e2d2 --- /dev/null +++ b/tests/queries/0_stateless/format_schemas/02566_ipv4_ipv6.capnp @@ -0,0 +1,6 @@ +@0xb6ecde1cd54a101d; + +struct Message { + ipv4 @0 :UInt32; + ipv6 @1 :Data; +} diff --git a/tests/queries/1_stateful/00173_group_by_use_nulls.reference b/tests/queries/1_stateful/00173_group_by_use_nulls.reference index 02723bf14dd..e82b996ad3c 100644 --- a/tests/queries/1_stateful/00173_group_by_use_nulls.reference +++ b/tests/queries/1_stateful/00173_group_by_use_nulls.reference @@ -8,3 +8,25 @@ 59183 1336 33010362 1336 800784 1336 +-- { echoOn } +set allow_experimental_analyzer = 1; +SELECT + CounterID AS k, + quantileBFloat16(0.5)(ResolutionWidth) +FROM remote('127.0.0.{1,2}', test, hits) +GROUP BY k +ORDER BY + count() DESC, + CounterID ASC +LIMIT 10 +SETTINGS group_by_use_nulls = 1; +1704509 1384 +732797 1336 +598875 1384 +792887 1336 +3807842 1336 +25703952 1336 +716829 1384 +59183 1336 +33010362 1336 +800784 1336 diff --git a/tests/queries/1_stateful/00173_group_by_use_nulls.sql b/tests/queries/1_stateful/00173_group_by_use_nulls.sql index 7acacc4e579..8531e9efaf8 100644 --- a/tests/queries/1_stateful/00173_group_by_use_nulls.sql +++ b/tests/queries/1_stateful/00173_group_by_use_nulls.sql @@ -8,3 +8,28 @@ ORDER BY CounterID ASC LIMIT 10 SETTINGS group_by_use_nulls = 1; + +SELECT + CounterID AS k, + quantileBFloat16(0.5)(ResolutionWidth) +FROM test.hits +GROUP BY k +ORDER BY + count() DESC, + CounterID ASC +LIMIT 10 +SETTINGS group_by_use_nulls = 1 FORMAT Null; + +-- { echoOn } +set allow_experimental_analyzer = 1; + +SELECT + CounterID AS k, + quantileBFloat16(0.5)(ResolutionWidth) +FROM remote('127.0.0.{1,2}', test, hits) +GROUP BY k +ORDER BY + count() DESC, + CounterID ASC +LIMIT 10 +SETTINGS group_by_use_nulls = 1; diff --git a/tests/queries/shell_config.sh b/tests/queries/shell_config.sh index 3006b74d3f9..ef70c82aefc 100644 --- a/tests/queries/shell_config.sh +++ b/tests/queries/shell_config.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# shellcheck disable=SC2120 # Don't check for ODR violation, since we may test shared build with ASAN export ASAN_OPTIONS=detect_odr_violation=0 @@ -136,12 +137,13 @@ function clickhouse_client_removed_host_parameter() function wait_for_queries_to_finish() { + local max_tries="${1:-20}" # Wait for all queries to finish (query may still be running if thread is killed by timeout) num_tries=0 while [[ $($CLICKHOUSE_CLIENT -q "SELECT count() FROM system.processes WHERE current_database=currentDatabase() AND query NOT LIKE '%system.processes%'") -ne 0 ]]; do sleep 0.5; num_tries=$((num_tries+1)) - if [ $num_tries -eq 20 ]; then + if [ $num_tries -eq $max_tries ]; then $CLICKHOUSE_CLIENT -q "SELECT * FROM system.processes WHERE current_database=currentDatabase() AND query NOT LIKE '%system.processes%' FORMAT Vertical" break fi diff --git a/utils/check-mysql-binlog/main.cpp b/utils/check-mysql-binlog/main.cpp index 7dd387ba5be..68558340180 100644 --- a/utils/check-mysql-binlog/main.cpp +++ b/utils/check-mysql-binlog/main.cpp @@ -17,7 +17,8 @@ static DB::MySQLReplication::BinlogEventPtr parseSingleEventBody( std::shared_ptr & last_table_map_event, bool exist_checksum) { DB::MySQLReplication::BinlogEventPtr event; - DB::ReadBufferPtr limit_read_buffer = std::make_shared(payload, header.event_size - 19, false); + DB::ReadBufferPtr limit_read_buffer = std::make_shared(payload, header.event_size - 19, + /* trow_exception */ false, /* exact_limit */ std::nullopt); DB::ReadBufferPtr event_payload = std::make_shared(*limit_read_buffer, exist_checksum ? 4 : 0); switch (header.type) diff --git a/utils/check-style/aspell-ignore/en/aspell-dict.txt b/utils/check-style/aspell-ignore/en/aspell-dict.txt index 1ad7432a5bf..f1bba4dc2fc 100644 --- a/utils/check-style/aspell-ignore/en/aspell-dict.txt +++ b/utils/check-style/aspell-ignore/en/aspell-dict.txt @@ -1,4 +1,4 @@ -personal_ws-1.1 en 484 +personal_ws-1.1 en 543 AArch ACLs AMQP @@ -25,6 +25,7 @@ CentOS ClickHouse ClickHouse's CodeBlock +CodeLLDB Config ConnectionDetails Contrib @@ -56,9 +57,8 @@ Hostname IPv IntN Integrations -invariants -JSONAsString JSONAsObject +JSONAsString JSONColumns JSONColumnsWithMetadata JSONCompact @@ -82,6 +82,7 @@ Jemalloc Jepsen KDevelop LGPL +LLDB LOCALTIME LOCALTIMESTAMP LibFuzzer @@ -104,10 +105,11 @@ NULLIF NVME NYPD NuRaft -ObjectId -Ok OLAP OLTP +ObjectId +Ok +OpenSSL OpenSUSE OpenStack OpenTelemetry @@ -128,10 +130,10 @@ PrettySpaceNoEscapes PrettySpaceNoEscapesMonoBlock Protobuf ProtobufSingle +QEMU QTCreator QueryCacheHits QueryCacheMisses -QEMU RBAC RawBLOB RedHat @@ -178,14 +180,14 @@ Valgrind Vectorized VirtualBox Werror +WithNamesAndTypes Woboq WriteBuffer WriteBuffers -WithNamesAndTypes XCode YAML -Yasm YYYY +Yasm Zipkin ZooKeeper ZooKeeper's @@ -200,6 +202,7 @@ autostart avro avx aws +backend backoff backticks benchmarking @@ -309,6 +312,7 @@ instantiation integrational integrations interserver +invariants jdbc jemalloc json @@ -333,8 +337,8 @@ jsonstringseachrowwithprogress kafka kafkacat konsole -latencies laion +latencies lexicographically libFuzzer libc @@ -373,9 +377,10 @@ mutex mysql mysqldump mysqljs +natively noop -nullable nullability +nullable num obfuscator odbc @@ -532,6 +537,7 @@ xcode xml xz zLib +zLinux zkcopy zlib znodes diff --git a/utils/checksum-for-compressed-block/main.cpp b/utils/checksum-for-compressed-block/main.cpp index 27a2154340e..4f9923e7638 100644 --- a/utils/checksum-for-compressed-block/main.cpp +++ b/utils/checksum-for-compressed-block/main.cpp @@ -2,7 +2,7 @@ #include #include #include -#include +#include /** A tool to easily prove if "Checksum doesn't match: corrupted data" diff --git a/utils/list-versions/version_date.tsv b/utils/list-versions/version_date.tsv index 3814e94bf24..c2d9781177d 100644 --- a/utils/list-versions/version_date.tsv +++ b/utils/list-versions/version_date.tsv @@ -1,7 +1,14 @@ +v23.2.4.12-stable 2023-03-10 +v23.2.3.17-stable 2023-03-06 +v23.2.2.20-stable 2023-03-01 v23.2.1.2537-stable 2023-02-23 +v23.1.5.24-stable 2023-03-10 +v23.1.4.58-stable 2023-03-01 v23.1.3.5-stable 2023-02-03 v23.1.2.9-stable 2023-01-29 v23.1.1.3077-stable 2023-01-25 +v22.12.5.34-stable 2023-03-10 +v22.12.4.76-stable 2023-03-01 v22.12.3.5-stable 2023-01-10 v22.12.2.25-stable 2023-01-06 v22.12.1.1752-stable 2022-12-15 @@ -25,6 +32,8 @@ v22.9.4.32-stable 2022-10-26 v22.9.3.18-stable 2022-09-30 v22.9.2.7-stable 2022-09-23 v22.9.1.2603-stable 2022-09-22 +v22.8.15.23-lts 2023-03-10 +v22.8.14.53-lts 2023-02-27 v22.8.13.20-lts 2023-01-29 v22.8.12.45-lts 2023-01-10 v22.8.11.15-lts 2022-12-08 diff --git a/utils/self-extracting-executable/decompressor.cpp b/utils/self-extracting-executable/decompressor.cpp index 37fbd043814..d41b9b1ebe1 100644 --- a/utils/self-extracting-executable/decompressor.cpp +++ b/utils/self-extracting-executable/decompressor.cpp @@ -168,6 +168,10 @@ int decompress(char * input, char * output, off_t start, off_t end, size_t max_n return 0; } +bool isSudo() +{ + return geteuid() == 0; +} /// Read data about files and decomrpess them. int decompressFiles(int input_fd, char * path, char * name, bool & have_compressed_analoge, bool & has_exec, char * decompressed_suffix, uint64_t * decompressed_umask) @@ -220,6 +224,8 @@ int decompressFiles(int input_fd, char * path, char * name, bool & have_compress return 1; } + bool is_sudo = isSudo(); + FileData file_info; /// Decompress files with appropriate file names for (size_t i = 0; i < le64toh(metadata.number_of_files); ++i) @@ -319,6 +325,9 @@ int decompressFiles(int input_fd, char * path, char * name, bool & have_compress perror("fsync"); if (0 != close(output_fd)) perror("close"); + + if (is_sudo) + chown(file_name, info_in.st_uid, info_in.st_gid); } if (0 != munmap(input, info_in.st_size)) @@ -414,6 +423,13 @@ int main(int/* argc*/, char* argv[]) else name = file_path; + struct stat input_info; + if (0 != stat(self, &input_info)) + { + perror("stat"); + return 1; + } + #if !defined(OS_DARWIN) && !defined(OS_FREEBSD) /// get inode of this executable uint64_t inode = getInode(self); @@ -441,13 +457,6 @@ int main(int/* argc*/, char* argv[]) return 1; } - struct stat input_info; - if (0 != stat(self, &input_info)) - { - perror("stat"); - return 1; - } - /// inconsistency in WSL1 Ubuntu - inode reported in /proc/self/maps is a 64bit to /// 32bit conversion of input_info.st_ino if (input_info.st_ino & 0xFFFFFFFF00000000 && !(inode & 0xFFFFFFFF00000000)) @@ -532,6 +541,9 @@ int main(int/* argc*/, char* argv[]) return 1; } + if (isSudo()) + chown(static_cast(self), input_info.st_uid, input_info.st_gid); + if (has_exec) { #if !defined(OS_DARWIN) && !defined(OS_FREEBSD) diff --git a/utils/wikistat-loader/main.cpp b/utils/wikistat-loader/main.cpp index 31ade014c74..493f1df05da 100644 --- a/utils/wikistat-loader/main.cpp +++ b/utils/wikistat-loader/main.cpp @@ -1,6 +1,6 @@ #include -#include +#include #include #include #include